You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2021/04/01 00:01:38 UTC
[flink] branch master updated (78a6dd6 -> 75e1f60)
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git.
from 78a6dd6 [hotfix][tests] Remove unused code.
new 60b3895 [FLINK-21976] Remove all files under flink-ml-parent
new a65b3ed [FLINK-21976] Remove fink-ml-* related configs from pom.xml
new 2519c10 [FLINK-21976] Remove Flink ML related examples from flink-examples
new b6b16eb [FLINK-21976] Remove python packages under flink-python/pyflink/ml
new 75e1f60 [FLINK-21976] Remove pyflink/ml related configs from python scripts
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
flink-dist/pom.xml | 7 -
flink-dist/src/main/assemblies/opt.xml | 8 -
.../flink/examples/java/ml/LinearRegression.java | 257 ---------
.../java/ml/util/LinearRegressionData.java | 69 ---
.../ml/util/LinearRegressionDataGenerator.java | 115 ----
.../flink/examples/scala/ml/LinearRegression.scala | 159 ------
flink-examples/flink-examples-streaming/pom.xml | 25 -
.../examples/ml/IncrementalLearningSkeleton.java | 212 --------
.../ml/util/IncrementalLearningSkeletonData.java | 33 --
.../examples/ml/IncrementalLearningSkeleton.scala | 184 -------
.../streaming/test/StreamingExamplesITCase.java | 9 -
.../scala/examples/StreamingExamplesITCase.scala | 9 -
flink-ml-parent/flink-ml-api/pom.xml | 47 --
.../org/apache/flink/ml/api/core/Estimator.java | 47 --
.../java/org/apache/flink/ml/api/core/Model.java | 37 --
.../org/apache/flink/ml/api/core/Pipeline.java | 259 ---------
.../apache/flink/ml/api/core/PipelineStage.java | 46 --
.../org/apache/flink/ml/api/core/Transformer.java | 42 --
.../apache/flink/ml/api/misc/param/ParamInfo.java | 151 ------
.../flink/ml/api/misc/param/ParamInfoFactory.java | 134 -----
.../flink/ml/api/misc/param/ParamValidator.java | 39 --
.../org/apache/flink/ml/api/misc/param/Params.java | 277 ----------
.../apache/flink/ml/api/misc/param/WithParams.java | 60 ---
.../flink/ml/util/param/ExtractParamInfosUtil.java | 71 ---
.../org/apache/flink/ml/api/core/PipelineTest.java | 167 ------
.../org/apache/flink/ml/api/misc/ParamsTest.java | 179 -------
.../ml/util/param/ExtractParamInfosUtilTest.java | 109 ----
flink-ml-parent/flink-ml-lib/pom.xml | 86 ---
.../org/apache/flink/ml/common/MLEnvironment.java | 151 ------
.../flink/ml/common/MLEnvironmentFactory.java | 116 -----
.../org/apache/flink/ml/common/linalg/BLAS.java | 234 ---------
.../apache/flink/ml/common/linalg/DenseMatrix.java | 577 ---------------------
.../apache/flink/ml/common/linalg/DenseVector.java | 379 --------------
.../apache/flink/ml/common/linalg/MatVecOp.java | 307 -----------
.../flink/ml/common/linalg/SparseVector.java | 574 --------------------
.../org/apache/flink/ml/common/linalg/Vector.java | 89 ----
.../flink/ml/common/linalg/VectorIterator.java | 73 ---
.../apache/flink/ml/common/linalg/VectorUtil.java | 240 ---------
.../org/apache/flink/ml/common/mapper/Mapper.java | 79 ---
.../flink/ml/common/mapper/MapperAdapter.java | 46 --
.../apache/flink/ml/common/mapper/ModelMapper.java | 66 ---
.../flink/ml/common/mapper/ModelMapperAdapter.java | 62 ---
.../common/model/BroadcastVariableModelSource.java | 47 --
.../apache/flink/ml/common/model/ModelSource.java | 40 --
.../flink/ml/common/model/RowsModelSource.java | 46 --
.../basicstatistic/MultivariateGaussian.java | 137 -----
.../ml/common/utils/DataSetConversionUtil.java | 167 ------
.../ml/common/utils/DataStreamConversionUtil.java | 167 ------
.../flink/ml/common/utils/OutputColsHelper.java | 211 --------
.../apache/flink/ml/common/utils/TableUtil.java | 439 ----------------
.../apache/flink/ml/common/utils/VectorTypes.java | 43 --
.../org/apache/flink/ml/operator/AlgoOperator.java | 186 -------
.../flink/ml/operator/batch/BatchOperator.java | 113 ----
.../operator/batch/source/TableSourceBatchOp.java | 40 --
.../flink/ml/operator/stream/StreamOperator.java | 114 ----
.../stream/source/TableSourceStreamOp.java | 40 --
.../flink/ml/params/shared/HasMLEnvironmentId.java | 43 --
.../ml/params/shared/colname/HasOutputCol.java | 48 --
.../shared/colname/HasOutputColDefaultAsNull.java | 49 --
.../ml/params/shared/colname/HasOutputCols.java | 48 --
.../shared/colname/HasOutputColsDefaultAsNull.java | 49 --
.../ml/params/shared/colname/HasPredictionCol.java | 42 --
.../shared/colname/HasPredictionDetailCol.java | 47 --
.../ml/params/shared/colname/HasReservedCols.java | 45 --
.../ml/params/shared/colname/HasSelectedCol.java | 48 --
.../colname/HasSelectedColDefaultAsNull.java | 49 --
.../ml/params/shared/colname/HasSelectedCols.java | 48 --
.../colname/HasSelectedColsDefaultAsNull.java | 49 --
.../apache/flink/ml/pipeline/EstimatorBase.java | 103 ----
.../org/apache/flink/ml/pipeline/ModelBase.java | 68 ---
.../flink/ml/pipeline/PipelineStageBase.java | 72 ---
.../apache/flink/ml/pipeline/TransformerBase.java | 99 ----
.../apache/flink/ml/common/MLEnvironmentTest.java | 83 ---
.../apache/flink/ml/common/linalg/BLASTest.java | 186 -------
.../flink/ml/common/linalg/DenseMatrixTest.java | 195 -------
.../flink/ml/common/linalg/DenseVectorTest.java | 158 ------
.../flink/ml/common/linalg/MatVecOpTest.java | 103 ----
.../flink/ml/common/linalg/SparseVectorTest.java | 232 ---------
.../flink/ml/common/linalg/VectorUtilTest.java | 76 ---
.../basicstatistic/MultivariateGaussianTest.java | 72 ---
.../ml/common/utils/DataSetConversionUtilTest.java | 200 -------
.../common/utils/DataStreamConversionUtilTest.java | 208 --------
.../ml/common/utils/OutputColsHelperTest.java | 249 ---------
.../flink/ml/common/utils/TableUtilTest.java | 200 -------
.../flink/ml/common/utils/VectorTypesTest.java | 78 ---
.../flink/ml/pipeline/EstimatorBaseTest.java | 89 ----
.../flink/ml/pipeline/PipelineStageTestBase.java | 70 ---
.../flink/ml/pipeline/TransformerBaseTest.java | 88 ----
.../ml/pipeline/UserDefinedPipelineStages.java | 58 ---
flink-ml-parent/flink-ml-uber/pom.xml | 76 ---
.../src/main/resources/META-INF/NOTICE | 10 -
.../main/resources/META-INF/licenses/LICENSE.core | 49 --
flink-ml-parent/pom.xml | 42 --
flink-python/README.md | 2 +-
flink-python/docs/index.rst | 1 -
flink-python/docs/pyflink.ml.rst | 52 --
flink-python/pyflink/ml/__init__.py | 17 -
flink-python/pyflink/ml/api/__init__.py | 28 -
flink-python/pyflink/ml/api/base.py | 348 -------------
flink-python/pyflink/ml/api/ml_environment.py | 92 ----
.../pyflink/ml/api/ml_environment_factory.py | 125 -----
flink-python/pyflink/ml/api/param/__init__.py | 21 -
flink-python/pyflink/ml/api/param/base.py | 350 -------------
flink-python/pyflink/ml/lib/__init__.py | 17 -
flink-python/pyflink/ml/lib/param/__init__.py | 21 -
flink-python/pyflink/ml/lib/param/colname.py | 78 ---
flink-python/pyflink/ml/tests/__init__.py | 17 -
.../pyflink/ml/tests/test_ml_environment.py | 55 --
.../ml/tests/test_ml_environment_factory.py | 48 --
flink-python/pyflink/ml/tests/test_params.py | 187 -------
flink-python/pyflink/ml/tests/test_pipeline.py | 171 ------
.../pyflink/ml/tests/test_pipeline_it_case.py | 189 -------
.../pyflink/ml/tests/test_pipeline_stage.py | 90 ----
flink-python/pyflink/pyflink_gateway_server.py | 2 -
flink-python/pyflink/shell.py | 2 -
flink-python/pyflink/testing/test_case_utils.py | 34 --
flink-python/setup.py | 7 +-
flink-python/tox.ini | 2 +-
pom.xml | 1 -
119 files changed, 3 insertions(+), 13004 deletions(-)
delete mode 100644 flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/LinearRegression.java
delete mode 100644 flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionData.java
delete mode 100644 flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionDataGenerator.java
delete mode 100644 flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/ml/LinearRegression.scala
delete mode 100644 flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
delete mode 100644 flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/util/IncrementalLearningSkeletonData.java
delete mode 100644 flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/ml/IncrementalLearningSkeleton.scala
delete mode 100644 flink-ml-parent/flink-ml-api/pom.xml
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Estimator.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Model.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Pipeline.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/PipelineStage.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Transformer.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfo.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfoFactory.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamValidator.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/Params.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/WithParams.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/util/param/ExtractParamInfosUtil.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/core/PipelineTest.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/misc/ParamsTest.java
delete mode 100644 flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/util/param/ExtractParamInfosUtilTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/pom.xml
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/BLAS.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseMatrix.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseVector.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/MatVecOp.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/SparseVector.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/Vector.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorIterator.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorUtil.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/Mapper.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/MapperAdapter.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapper.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapperAdapter.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/BroadcastVariableModelSource.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/ModelSource.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/RowsModelSource.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussian.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataSetConversionUtil.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataStreamConversionUtil.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/OutputColsHelper.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/TableUtil.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/VectorTypes.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCol.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColDefaultAsNull.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCols.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColsDefaultAsNull.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionCol.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionDetailCol.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasReservedCols.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCol.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColDefaultAsNull.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCols.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColsDefaultAsNull.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/BLASTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseMatrixTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseVectorTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/MatVecOpTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/SparseVectorTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/VectorUtilTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussianTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataSetConversionUtilTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataStreamConversionUtilTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/OutputColsHelperTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/TableUtilTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/VectorTypesTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/EstimatorBaseTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/PipelineStageTestBase.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/TransformerBaseTest.java
delete mode 100644 flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/pipeline/UserDefinedPipelineStages.java
delete mode 100644 flink-ml-parent/flink-ml-uber/pom.xml
delete mode 100644 flink-ml-parent/flink-ml-uber/src/main/resources/META-INF/NOTICE
delete mode 100644 flink-ml-parent/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.core
delete mode 100644 flink-ml-parent/pom.xml
delete mode 100644 flink-python/docs/pyflink.ml.rst
delete mode 100644 flink-python/pyflink/ml/__init__.py
delete mode 100644 flink-python/pyflink/ml/api/__init__.py
delete mode 100644 flink-python/pyflink/ml/api/base.py
delete mode 100644 flink-python/pyflink/ml/api/ml_environment.py
delete mode 100644 flink-python/pyflink/ml/api/ml_environment_factory.py
delete mode 100644 flink-python/pyflink/ml/api/param/__init__.py
delete mode 100644 flink-python/pyflink/ml/api/param/base.py
delete mode 100644 flink-python/pyflink/ml/lib/__init__.py
delete mode 100644 flink-python/pyflink/ml/lib/param/__init__.py
delete mode 100644 flink-python/pyflink/ml/lib/param/colname.py
delete mode 100644 flink-python/pyflink/ml/tests/__init__.py
delete mode 100644 flink-python/pyflink/ml/tests/test_ml_environment.py
delete mode 100644 flink-python/pyflink/ml/tests/test_ml_environment_factory.py
delete mode 100644 flink-python/pyflink/ml/tests/test_params.py
delete mode 100644 flink-python/pyflink/ml/tests/test_pipeline.py
delete mode 100644 flink-python/pyflink/ml/tests/test_pipeline_it_case.py
delete mode 100644 flink-python/pyflink/ml/tests/test_pipeline_stage.py
[flink] 03/05: [FLINK-21976] Remove Flink ML related examples from
flink-examples
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 2519c1097434fd67081f82a26d13ad93bf31800e
Author: Dong Lin <li...@gmail.com>
AuthorDate: Wed Mar 31 12:10:07 2021 +0800
[FLINK-21976] Remove Flink ML related examples from flink-examples
This closes #15405.
---
.../flink/examples/java/ml/LinearRegression.java | 257 ---------------------
.../java/ml/util/LinearRegressionData.java | 69 ------
.../ml/util/LinearRegressionDataGenerator.java | 115 ---------
.../flink/examples/scala/ml/LinearRegression.scala | 159 -------------
flink-examples/flink-examples-streaming/pom.xml | 25 --
.../examples/ml/IncrementalLearningSkeleton.java | 212 -----------------
.../ml/util/IncrementalLearningSkeletonData.java | 33 ---
.../examples/ml/IncrementalLearningSkeleton.scala | 184 ---------------
.../streaming/test/StreamingExamplesITCase.java | 9 -
.../scala/examples/StreamingExamplesITCase.scala | 9 -
10 files changed, 1072 deletions(-)
diff --git a/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/LinearRegression.java b/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/LinearRegression.java
deleted file mode 100644
index 4f2f528..0000000
--- a/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/LinearRegression.java
+++ /dev/null
@@ -1,257 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.java.ml;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.functions.ReduceFunction;
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.operators.IterativeDataSet;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.examples.java.ml.util.LinearRegressionData;
-
-import java.io.Serializable;
-import java.util.Collection;
-
-/**
- * This example implements a basic Linear Regression to solve the y = theta0 + theta1*x problem
- * using batch gradient descent algorithm.
- *
- * <p>Linear Regression with BGD(batch gradient descent) algorithm is an iterative clustering
- * algorithm and works as follows:<br>
- * Giving a data set and target set, the BGD try to find out the best parameters for the data set to
- * fit the target set. In each iteration, the algorithm computes the gradient of the cost function
- * and use it to update all the parameters. The algorithm terminates after a fixed number of
- * iterations (as in this implementation) With enough iteration, the algorithm can minimize the cost
- * function and find the best parameters This is the Wikipedia entry for the <a href =
- * "http://en.wikipedia.org/wiki/Linear_regression">Linear regression</a> and <a href =
- * "http://en.wikipedia.org/wiki/Gradient_descent">Gradient descent algorithm</a>.
- *
- * <p>This implementation works on one-dimensional data. And find the two-dimensional theta.<br>
- * It find the best Theta parameter to fit the target.
- *
- * <p>Input files are plain text files and must be formatted as follows:
- *
- * <ul>
- * <li>Data points are represented as two double values separated by a blank character. The first
- * one represent the X(the training data) and the second represent the Y(target). Data points
- * are separated by newline characters.<br>
- * For example <code>"-0.02 -0.04\n5.3 10.6\n"</code> gives two data points (x=-0.02, y=-0.04)
- * and (x=5.3, y=10.6).
- * </ul>
- *
- * <p>This example shows how to use:
- *
- * <ul>
- * <li>Bulk iterations
- * <li>Broadcast variables in bulk iterations
- * <li>Custom Java objects (PoJos)
- * </ul>
- */
-@SuppressWarnings("serial")
-public class LinearRegression {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- final ParameterTool params = ParameterTool.fromArgs(args);
-
- // set up execution environment
- final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
- final int iterations = params.getInt("iterations", 10);
-
- // make parameters available in the web interface
- env.getConfig().setGlobalJobParameters(params);
-
- // get input x data from elements
- DataSet<Data> data;
- if (params.has("input")) {
- // read data from CSV file
- data =
- env.readCsvFile(params.get("input"))
- .fieldDelimiter(" ")
- .includeFields(true, true)
- .pojoType(Data.class);
- } else {
- System.out.println("Executing LinearRegression example with default input data set.");
- System.out.println("Use --input to specify file input.");
- data = LinearRegressionData.getDefaultDataDataSet(env);
- }
-
- // get the parameters from elements
- DataSet<Params> parameters = LinearRegressionData.getDefaultParamsDataSet(env);
-
- // set number of bulk iterations for SGD linear Regression
- IterativeDataSet<Params> loop = parameters.iterate(iterations);
-
- DataSet<Params> newParameters =
- data
- // compute a single step using every sample
- .map(new SubUpdate())
- .withBroadcastSet(loop, "parameters")
- // sum up all the steps
- .reduce(new UpdateAccumulator())
- // average the steps and update all parameters
- .map(new Update());
-
- // feed new parameters back into next iteration
- DataSet<Params> result = loop.closeWith(newParameters);
-
- // emit result
- if (params.has("output")) {
- result.writeAsText(params.get("output"));
- // execute program
- env.execute("Linear Regression example");
- } else {
- System.out.println("Printing result to stdout. Use --output to specify output path.");
- result.print();
- }
- }
-
- // *************************************************************************
- // DATA TYPES
- // *************************************************************************
-
- /** A simple data sample, x means the input, and y means the target. */
- public static class Data implements Serializable {
- public double x, y;
-
- public Data() {}
-
- public Data(double x, double y) {
- this.x = x;
- this.y = y;
- }
-
- @Override
- public String toString() {
- return "(" + x + "|" + y + ")";
- }
- }
-
- /** A set of parameters -- theta0, theta1. */
- public static class Params implements Serializable {
-
- private double theta0, theta1;
-
- public Params() {}
-
- public Params(double x0, double x1) {
- this.theta0 = x0;
- this.theta1 = x1;
- }
-
- @Override
- public String toString() {
- return theta0 + " " + theta1;
- }
-
- public double getTheta0() {
- return theta0;
- }
-
- public double getTheta1() {
- return theta1;
- }
-
- public void setTheta0(double theta0) {
- this.theta0 = theta0;
- }
-
- public void setTheta1(double theta1) {
- this.theta1 = theta1;
- }
-
- public Params div(Integer a) {
- this.theta0 = theta0 / a;
- this.theta1 = theta1 / a;
- return this;
- }
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /** Compute a single BGD type update for every parameters. */
- public static class SubUpdate extends RichMapFunction<Data, Tuple2<Params, Integer>> {
-
- private Collection<Params> parameters;
-
- private Params parameter;
-
- private int count = 1;
-
- /** Reads the parameters from a broadcast variable into a collection. */
- @Override
- public void open(Configuration parameters) throws Exception {
- this.parameters = getRuntimeContext().getBroadcastVariable("parameters");
- }
-
- @Override
- public Tuple2<Params, Integer> map(Data in) throws Exception {
-
- for (Params p : parameters) {
- this.parameter = p;
- }
-
- double theta0 =
- parameter.theta0
- - 0.01 * ((parameter.theta0 + (parameter.theta1 * in.x)) - in.y);
- double theta1 =
- parameter.theta1
- - 0.01
- * (((parameter.theta0 + (parameter.theta1 * in.x)) - in.y)
- * in.x);
-
- return new Tuple2<Params, Integer>(new Params(theta0, theta1), count);
- }
- }
-
- /** Accumulator all the update. */
- public static class UpdateAccumulator implements ReduceFunction<Tuple2<Params, Integer>> {
-
- @Override
- public Tuple2<Params, Integer> reduce(
- Tuple2<Params, Integer> val1, Tuple2<Params, Integer> val2) {
-
- double newTheta0 = val1.f0.theta0 + val2.f0.theta0;
- double newTheta1 = val1.f0.theta1 + val2.f0.theta1;
- Params result = new Params(newTheta0, newTheta1);
- return new Tuple2<Params, Integer>(result, val1.f1 + val2.f1);
- }
- }
-
- /** Compute the final update by average them. */
- public static class Update implements MapFunction<Tuple2<Params, Integer>, Params> {
-
- @Override
- public Params map(Tuple2<Params, Integer> arg0) throws Exception {
-
- return arg0.f0.div(arg0.f1);
- }
- }
-}
diff --git a/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionData.java b/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionData.java
deleted file mode 100644
index 5cb1339..0000000
--- a/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionData.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.java.ml.util;
-
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.examples.java.ml.LinearRegression.Data;
-import org.apache.flink.examples.java.ml.LinearRegression.Params;
-
-import java.util.LinkedList;
-import java.util.List;
-
-/**
- * Provides the default data sets used for the Linear Regression example program. The default data
- * sets are used, if no parameters are given to the program.
- */
-public class LinearRegressionData {
-
- // We have the data as object arrays so that we can also generate Scala Data
- // Sources from it.
- public static final Object[][] PARAMS = new Object[][] {new Object[] {0.0, 0.0}};
-
- public static final Object[][] DATA =
- new Object[][] {
- new Object[] {0.5, 1.0}, new Object[] {1.0, 2.0},
- new Object[] {2.0, 4.0}, new Object[] {3.0, 6.0},
- new Object[] {4.0, 8.0}, new Object[] {5.0, 10.0},
- new Object[] {6.0, 12.0}, new Object[] {7.0, 14.0},
- new Object[] {8.0, 16.0}, new Object[] {9.0, 18.0},
- new Object[] {10.0, 20.0}, new Object[] {-0.08, -0.16},
- new Object[] {0.13, 0.26}, new Object[] {-1.17, -2.35},
- new Object[] {1.72, 3.45}, new Object[] {1.70, 3.41},
- new Object[] {1.20, 2.41}, new Object[] {-0.59, -1.18},
- new Object[] {0.28, 0.57}, new Object[] {1.65, 3.30},
- new Object[] {-0.55, -1.08}
- };
-
- public static DataSet<Params> getDefaultParamsDataSet(ExecutionEnvironment env) {
- List<Params> paramsList = new LinkedList<>();
- for (Object[] params : PARAMS) {
- paramsList.add(new Params((Double) params[0], (Double) params[1]));
- }
- return env.fromCollection(paramsList);
- }
-
- public static DataSet<Data> getDefaultDataDataSet(ExecutionEnvironment env) {
- List<Data> dataList = new LinkedList<>();
- for (Object[] data : DATA) {
- dataList.add(new Data((Double) data[0], (Double) data[1]));
- }
- return env.fromCollection(dataList);
- }
-}
diff --git a/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionDataGenerator.java b/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionDataGenerator.java
deleted file mode 100644
index 52a912f..0000000
--- a/flink-examples/flink-examples-batch/src/main/java/org/apache/flink/examples/java/ml/util/LinearRegressionDataGenerator.java
+++ /dev/null
@@ -1,115 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.java.ml.util;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.text.DecimalFormat;
-import java.util.Locale;
-import java.util.Random;
-
-/**
- * Generates data for the {@link org.apache.flink.examples.java.ml.LinearRegression} example
- * program.
- */
-public class LinearRegressionDataGenerator {
-
- static {
- Locale.setDefault(Locale.US);
- }
-
- private static final String POINTS_FILE = "data";
- private static final long DEFAULT_SEED = 4650285087650871364L;
- private static final int DIMENSIONALITY = 1;
- private static final DecimalFormat FORMAT = new DecimalFormat("#0.00");
- private static final char DELIMITER = ' ';
-
- /**
- * Main method to generate data for the {@link
- * org.apache.flink.examples.java.ml.LinearRegression} example program.
- *
- * <p>The generator creates to files:
- *
- * <ul>
- * <li><code>{tmp.dir}/data</code> for the data points
- * </ul>
- *
- * @param args
- * <ol>
- * <li>Int: Number of data points
- * <li><b>Optional</b> Long: Random seed
- * </ol>
- */
- public static void main(String[] args) throws IOException {
-
- // check parameter count
- if (args.length < 1) {
- System.out.println("LinearRegressionDataGenerator <numberOfDataPoints> [<seed>]");
- System.exit(1);
- }
-
- // parse parameters
- final int numDataPoints = Integer.parseInt(args[0]);
- final long firstSeed = args.length > 1 ? Long.parseLong(args[4]) : DEFAULT_SEED;
- final Random random = new Random(firstSeed);
- final String tmpDir = System.getProperty("java.io.tmpdir");
-
- // write the points out
- BufferedWriter pointsOut = null;
- try {
- pointsOut = new BufferedWriter(new FileWriter(new File(tmpDir + "/" + POINTS_FILE)));
- StringBuilder buffer = new StringBuilder();
-
- // DIMENSIONALITY + 1 means that the number of x(dimensionality) and target y
- double[] point = new double[DIMENSIONALITY + 1];
-
- for (int i = 1; i <= numDataPoints; i++) {
- point[0] = random.nextGaussian();
- point[1] = 2 * point[0] + 0.01 * random.nextGaussian();
- writePoint(point, buffer, pointsOut);
- }
-
- } finally {
- if (pointsOut != null) {
- pointsOut.close();
- }
- }
-
- System.out.println(
- "Wrote " + numDataPoints + " data points to " + tmpDir + "/" + POINTS_FILE);
- }
-
- private static void writePoint(double[] data, StringBuilder buffer, BufferedWriter out)
- throws IOException {
- buffer.setLength(0);
-
- // write coordinates
- for (int j = 0; j < data.length; j++) {
- buffer.append(FORMAT.format(data[j]));
- if (j < data.length - 1) {
- buffer.append(DELIMITER);
- }
- }
-
- out.write(buffer.toString());
- out.newLine();
- }
-}
diff --git a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/ml/LinearRegression.scala b/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/ml/LinearRegression.scala
deleted file mode 100644
index 4663db6..0000000
--- a/flink-examples/flink-examples-batch/src/main/scala/org/apache/flink/examples/scala/ml/LinearRegression.scala
+++ /dev/null
@@ -1,159 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.examples.scala.ml
-
-import org.apache.flink.api.common.functions._
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.api.scala._
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.examples.java.ml.util.LinearRegressionData
-
-import scala.collection.JavaConverters._
-
-/**
- * This example implements a basic Linear Regression to solve the y = theta0 + theta1*x problem
- * using batch gradient descent algorithm.
- *
- * Linear Regression with BGD(batch gradient descent) algorithm is an iterative algorithm and
- * works as follows:
- *
- * Giving a data set and target set, the BGD try to find out the best parameters for the data set
- * to fit the target set.
- * In each iteration, the algorithm computes the gradient of the cost function and use it to
- * update all the parameters.
- * The algorithm terminates after a fixed number of iterations (as in this implementation).
- * With enough iteration, the algorithm can minimize the cost function and find the best parameters
- * This is the Wikipedia entry for the
- * [[http://en.wikipedia.org/wiki/Linear_regression Linear regression]] and
- * [[http://en.wikipedia.org/wiki/Gradient_descent Gradient descent algorithm]].
- *
- * This implementation works on one-dimensional data and finds the best two-dimensional theta to
- * fit the target.
- *
- * Input files are plain text files and must be formatted as follows:
- *
- * - Data points are represented as two double values separated by a blank character. The first
- * one represent the X(the training data) and the second represent the Y(target). Data points are
- * separated by newline characters.
- * For example `"-0.02 -0.04\n5.3 10.6\n"`gives two data points
- * (x=-0.02, y=-0.04) and (x=5.3, y=10.6).
- *
- * This example shows how to use:
- *
- * - Bulk iterations
- * - Broadcast variables in bulk iterations
- */
-object LinearRegression {
-
- def main(args: Array[String]) {
-
- val params: ParameterTool = ParameterTool.fromArgs(args)
-
- // set up execution environment
- val env = ExecutionEnvironment.getExecutionEnvironment
-
- // make parameters available in the web interface
- env.getConfig.setGlobalJobParameters(params)
-
- val parameters = env.fromCollection(LinearRegressionData.PARAMS map {
- case Array(x, y) => Params(x.asInstanceOf[Double], y.asInstanceOf[Double])
- })
-
- val data =
- if (params.has("input")) {
- env.readCsvFile[(Double, Double)](
- params.get("input"),
- fieldDelimiter = " ",
- includedFields = Array(0, 1))
- .map { t => new Data(t._1, t._2) }
- } else {
- println("Executing LinearRegression example with default input data set.")
- println("Use --input to specify file input.")
- val data = LinearRegressionData.DATA map {
- case Array(x, y) => Data(x.asInstanceOf[Double], y.asInstanceOf[Double])
- }
- env.fromCollection(data)
- }
-
- val numIterations = params.getInt("iterations", 10)
-
- val result = parameters.iterate(numIterations) { currentParameters =>
- val newParameters = data
- .map(new SubUpdate).withBroadcastSet(currentParameters, "parameters")
- .reduce { (p1, p2) =>
- val result = p1._1 + p2._1
- (result, p1._2 + p2._2)
- }
- .map { x => x._1.div(x._2) }
- newParameters
- }
-
- if (params.has("output")) {
- result.writeAsText(params.get("output"))
- env.execute("Scala Linear Regression example")
- } else {
- println("Printing result to stdout. Use --output to specify output path.")
- result.print()
- }
- }
-
- /**
- * A simple data sample, x means the input, and y means the target.
- */
- case class Data(var x: Double, var y: Double)
-
- /**
- * A set of parameters -- theta0, theta1.
- */
- case class Params(theta0: Double, theta1: Double) {
- def div(a: Int): Params = {
- Params(theta0 / a, theta1 / a)
- }
-
- def + (other: Params) = {
- Params(theta0 + other.theta0, theta1 + other.theta1)
- }
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Compute a single BGD type update for every parameters.
- */
- class SubUpdate extends RichMapFunction[Data, (Params, Int)] {
-
- private var parameter: Params = null
-
- /** Reads the parameters from a broadcast variable into a collection. */
- override def open(parameters: Configuration) {
- val parameters = getRuntimeContext.getBroadcastVariable[Params]("parameters").asScala
- parameter = parameters.head
- }
-
- def map(in: Data): (Params, Int) = {
- val theta0 =
- parameter.theta0 - 0.01 * ((parameter.theta0 + (parameter.theta1 * in.x)) - in.y)
- val theta1 =
- parameter.theta1 - 0.01 * (((parameter.theta0 + (parameter.theta1 * in.x)) - in.y) * in.x)
- (Params(theta0, theta1), 1)
- }
- }
-}
diff --git a/flink-examples/flink-examples-streaming/pom.xml b/flink-examples/flink-examples-streaming/pom.xml
index 214cff5..c487a94 100644
--- a/flink-examples/flink-examples-streaming/pom.xml
+++ b/flink-examples/flink-examples-streaming/pom.xml
@@ -212,30 +212,6 @@ under the License.
</configuration>
</execution>
- <!-- IncrementalLearning -->
- <execution>
- <id>IncrementalLearning</id>
- <phase>package</phase>
- <goals>
- <goal>jar</goal>
- </goals>
- <configuration>
- <classifier>IncrementalLearning</classifier>
-
- <archive>
- <manifestEntries>
- <program-class>org.apache.flink.streaming.examples.ml.IncrementalLearningSkeleton</program-class>
- </manifestEntries>
- </archive>
-
- <includes>
- <include>org/apache/flink/streaming/examples/ml/*.class</include>
- <include>META-INF/LICENSE</include>
- <include>META-INF/NOTICE</include>
- </includes>
- </configuration>
- </execution>
-
<!-- WindowJoin -->
<execution>
<id>WindowJoin</id>
@@ -557,7 +533,6 @@ under the License.
<id>rename</id>
<configuration>
<target>
- <copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-IncrementalLearning.jar" tofile="${project.basedir}/target/IncrementalLearning.jar" />
<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-Iteration.jar" tofile="${project.basedir}/target/Iteration.jar" />
<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-SessionWindowing.jar" tofile="${project.basedir}/target/SessionWindowing.jar" />
<copy file="${project.basedir}/target/flink-examples-streaming_${scala.binary.version}-${project.version}-TopSpeedWindowing.jar" tofile="${project.basedir}/target/TopSpeedWindowing.jar" />
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
deleted file mode 100644
index 4f55f61..0000000
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/IncrementalLearningSkeleton.java
+++ /dev/null
@@ -1,212 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.ml;
-
-import org.apache.flink.api.java.utils.ParameterTool;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;
-import org.apache.flink.streaming.api.functions.co.CoMapFunction;
-import org.apache.flink.streaming.api.functions.source.SourceFunction;
-import org.apache.flink.streaming.api.functions.windowing.AllWindowFunction;
-import org.apache.flink.streaming.api.watermark.Watermark;
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows;
-import org.apache.flink.streaming.api.windowing.time.Time;
-import org.apache.flink.streaming.api.windowing.windows.TimeWindow;
-import org.apache.flink.util.Collector;
-
-/**
- * Skeleton for incremental machine learning algorithm consisting of a pre-computed model, which
- * gets updated for the new inputs and new input data for which the job provides predictions.
- *
- * <p>This may serve as a base of a number of algorithms, e.g. updating an incremental Alternating
- * Least Squares model while also providing the predictions.
- *
- * <p>This example shows how to use:
- *
- * <ul>
- * <li>Connected streams
- * <li>CoFunctions
- * <li>Tuple data types
- * </ul>
- */
-public class IncrementalLearningSkeleton {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- public static void main(String[] args) throws Exception {
-
- // Checking input parameters
- final ParameterTool params = ParameterTool.fromArgs(args);
-
- StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
-
- DataStream<Integer> trainingData = env.addSource(new FiniteTrainingDataSource());
- DataStream<Integer> newData = env.addSource(new FiniteNewDataSource());
-
- // build new model on every second of new data
- DataStream<Double[]> model =
- trainingData
- .assignTimestampsAndWatermarks(new LinearTimestamp())
- .windowAll(TumblingEventTimeWindows.of(Time.milliseconds(5000)))
- .apply(new PartialModelBuilder());
-
- // use partial model for newData
- DataStream<Integer> prediction = newData.connect(model).map(new Predictor());
-
- // emit result
- if (params.has("output")) {
- prediction.writeAsText(params.get("output"));
- } else {
- System.out.println("Printing result to stdout. Use --output to specify output path.");
- prediction.print();
- }
-
- // execute program
- env.execute("Streaming Incremental Learning");
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Feeds new data for newData. By default it is implemented as constantly emitting the Integer 1
- * in a loop.
- */
- public static class FiniteNewDataSource implements SourceFunction<Integer> {
- private static final long serialVersionUID = 1L;
- private int counter;
-
- @Override
- public void run(SourceContext<Integer> ctx) throws Exception {
- Thread.sleep(15);
- while (counter < 50) {
- ctx.collect(getNewData());
- }
- }
-
- @Override
- public void cancel() {
- // No cleanup needed
- }
-
- private Integer getNewData() throws InterruptedException {
- Thread.sleep(5);
- counter++;
- return 1;
- }
- }
-
- /**
- * Feeds new training data for the partial model builder. By default it is implemented as
- * constantly emitting the Integer 1 in a loop.
- */
- public static class FiniteTrainingDataSource implements SourceFunction<Integer> {
- private static final long serialVersionUID = 1L;
- private int counter = 0;
-
- @Override
- public void run(SourceContext<Integer> collector) throws Exception {
- while (counter < 8200) {
- collector.collect(getTrainingData());
- }
- }
-
- @Override
- public void cancel() {
- // No cleanup needed
- }
-
- private Integer getTrainingData() throws InterruptedException {
- counter++;
- return 1;
- }
- }
-
- private static class LinearTimestamp implements AssignerWithPunctuatedWatermarks<Integer> {
- private static final long serialVersionUID = 1L;
-
- private long counter = 0L;
-
- @Override
- public long extractTimestamp(Integer element, long previousElementTimestamp) {
- return counter += 10L;
- }
-
- @Override
- public Watermark checkAndGetNextWatermark(Integer lastElement, long extractedTimestamp) {
- return new Watermark(counter - 1);
- }
- }
-
- /** Builds up-to-date partial models on new training data. */
- public static class PartialModelBuilder
- implements AllWindowFunction<Integer, Double[], TimeWindow> {
- private static final long serialVersionUID = 1L;
-
- protected Double[] buildPartialModel(Iterable<Integer> values) {
- return new Double[] {1.};
- }
-
- @Override
- public void apply(TimeWindow window, Iterable<Integer> values, Collector<Double[]> out)
- throws Exception {
- out.collect(buildPartialModel(values));
- }
- }
-
- /**
- * Creates newData using the model produced in batch-processing and the up-to-date partial
- * model.
- *
- * <p>By default emits the Integer 0 for every newData and the Integer 1 for every model update.
- */
- public static class Predictor implements CoMapFunction<Integer, Double[], Integer> {
- private static final long serialVersionUID = 1L;
-
- Double[] batchModel = null;
- Double[] partialModel = null;
-
- @Override
- public Integer map1(Integer value) {
- // Return newData
- return predict(value);
- }
-
- @Override
- public Integer map2(Double[] value) {
- // Update model
- partialModel = value;
- batchModel = getBatchModel();
- return 1;
- }
-
- // pulls model built with batch-job on the old training data
- protected Double[] getBatchModel() {
- return new Double[] {0.};
- }
-
- // performs newData using the two models
- protected Integer predict(Integer inTuple) {
- return 0;
- }
- }
-}
diff --git a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/util/IncrementalLearningSkeletonData.java b/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/util/IncrementalLearningSkeletonData.java
deleted file mode 100644
index eaf5bdb..0000000
--- a/flink-examples/flink-examples-streaming/src/main/java/org/apache/flink/streaming/examples/ml/util/IncrementalLearningSkeletonData.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.examples.ml.util;
-
-/** Data for IncrementalLearningSkeletonITCase. */
-public class IncrementalLearningSkeletonData {
-
- public static final String RESULTS =
- "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n"
- + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "1\n" + "0\n" + "0\n" + "0\n" + "0\n"
- + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n"
- + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n"
- + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n"
- + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n"
- + "0\n" + "0\n" + "0\n" + "0\n" + "0\n" + "0\n";
-
- private IncrementalLearningSkeletonData() {}
-}
diff --git a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/ml/IncrementalLearningSkeleton.scala b/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/ml/IncrementalLearningSkeleton.scala
deleted file mode 100644
index 2f1e168..0000000
--- a/flink-examples/flink-examples-streaming/src/main/scala/org/apache/flink/streaming/scala/examples/ml/IncrementalLearningSkeleton.scala
+++ /dev/null
@@ -1,184 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.streaming.scala.examples.ml
-
-import org.apache.flink.api.java.utils.ParameterTool
-import org.apache.flink.api.scala._
-import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks
-import org.apache.flink.streaming.api.functions.co.CoMapFunction
-import org.apache.flink.streaming.api.functions.source.SourceFunction
-import org.apache.flink.streaming.api.functions.source.SourceFunction.SourceContext
-import org.apache.flink.streaming.api.scala.function.AllWindowFunction
-import org.apache.flink.streaming.api.scala.{DataStream, StreamExecutionEnvironment}
-import org.apache.flink.streaming.api.watermark.Watermark
-import org.apache.flink.streaming.api.windowing.assigners.TumblingEventTimeWindows
-import org.apache.flink.streaming.api.windowing.time.Time
-import org.apache.flink.streaming.api.windowing.windows.TimeWindow
-import org.apache.flink.util.Collector
-
-/**
- * Skeleton for incremental machine learning algorithm consisting of a
- * pre-computed model, which gets updated for the new inputs and new input data
- * for which the job provides predictions.
- *
- * This may serve as a base of a number of algorithms, e.g. updating an
- * incremental Alternating Least Squares model while also providing the
- * predictions.
- *
- * This example shows how to use:
- *
- * - Connected streams
- * - CoFunctions
- * - Tuple data types
- *
- */
-object IncrementalLearningSkeleton {
-
- // *************************************************************************
- // PROGRAM
- // *************************************************************************
-
- def main(args: Array[String]): Unit = {
- // Checking input parameters
- val params = ParameterTool.fromArgs(args)
-
- // set up the execution environment
- val env = StreamExecutionEnvironment.getExecutionEnvironment
-
- // build new model on every second of new data
- val trainingData: DataStream[Integer] = env.addSource(new FiniteTrainingDataSource)
- val newData: DataStream[Integer] = env.addSource(new FiniteNewDataSource)
-
- val model: DataStream[Array[java.lang.Double]] = trainingData
- .assignTimestampsAndWatermarks(new LinearTimestamp)
- .windowAll(TumblingEventTimeWindows.of(Time.milliseconds(5000)))
- .apply(new PartialModelBuilder)
-
- // use partial model for newData
- val prediction: DataStream[Integer] = newData.connect(model).map(new Predictor)
-
- // emit result
- if (params.has("output")) {
- prediction.writeAsText(params.get("output"))
- } else {
- println("Printing result to stdout. Use --output to specify output path.")
- prediction.print()
- }
-
- // execute program
- env.execute("Streaming Incremental Learning")
- }
-
- // *************************************************************************
- // USER FUNCTIONS
- // *************************************************************************
-
- /**
- * Feeds new data for newData. By default it is implemented as constantly
- * emitting the Integer 1 in a loop.
- */
- private class FiniteNewDataSource extends SourceFunction[Integer] {
- override def run(ctx: SourceContext[Integer]) = {
- Thread.sleep(15)
- (0 until 50).foreach{ _ =>
- Thread.sleep(5)
- ctx.collect(1)
- }
- }
-
- override def cancel() = {
- // No cleanup needed
- }
- }
-
- /**
- * Feeds new training data for the partial model builder. By default it is
- * implemented as constantly emitting the Integer 1 in a loop.
- */
- private class FiniteTrainingDataSource extends SourceFunction[Integer] {
- override def run(ctx: SourceContext[Integer]) =
- (0 until 8200).foreach( _ => ctx.collect(1) )
-
- override def cancel() = {
- // No cleanup needed
- }
- }
-
- private class LinearTimestamp extends AssignerWithPunctuatedWatermarks[Integer] {
- var counter = 0L
-
- override def extractTimestamp(element: Integer, previousElementTimestamp: Long): Long = {
- counter += 10L
- counter
- }
-
- override def checkAndGetNextWatermark(lastElement: Integer, extractedTimestamp: Long) = {
- new Watermark(counter - 1)
- }
- }
-
- /**
- * Builds up-to-date partial models on new training data.
- */
- private class PartialModelBuilder
- extends AllWindowFunction[Integer, Array[java.lang.Double], TimeWindow] {
-
- protected def buildPartialModel(values: Iterable[Integer]): Array[java.lang.Double] =
- Array[java.lang.Double](1)
-
- override def apply(window: TimeWindow,
- values: Iterable[Integer],
- out: Collector[Array[java.lang.Double]]): Unit = {
- out.collect(buildPartialModel(values))
- }
- }
-
- /**
- * Creates newData using the model produced in batch-processing and the
- * up-to-date partial model.
- *
- * By default emits the Integer 0 for every newData and the Integer 1
- * for every model update.
- *
- */
- private class Predictor extends CoMapFunction[Integer, Array[java.lang.Double], Integer] {
-
- var batchModel: Array[java.lang.Double] = null
- var partialModel: Array[java.lang.Double] = null
-
- override def map1(value: Integer): Integer = {
- // Return newData
- predict(value)
- }
-
- override def map2(value: Array[java.lang.Double]): Integer = {
- // Update model
- partialModel = value
- batchModel = getBatchModel()
- 1
- }
-
- // pulls model built with batch-job on the old training data
- protected def getBatchModel(): Array[java.lang.Double] = Array[java.lang.Double](0)
-
- // performs newData using the two models
- protected def predict(inTuple: Int): Int = 0
- }
-
-}
diff --git a/flink-examples/flink-examples-streaming/src/test/java/org/apache/flink/streaming/test/StreamingExamplesITCase.java b/flink-examples/flink-examples-streaming/src/test/java/org/apache/flink/streaming/test/StreamingExamplesITCase.java
index d8f8b56..7598f98 100644
--- a/flink-examples/flink-examples-streaming/src/test/java/org/apache/flink/streaming/test/StreamingExamplesITCase.java
+++ b/flink-examples/flink-examples-streaming/src/test/java/org/apache/flink/streaming/test/StreamingExamplesITCase.java
@@ -30,7 +30,6 @@ import org.apache.flink.core.fs.FileSystem;
import org.apache.flink.streaming.api.datastream.DataStream;
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
import org.apache.flink.streaming.examples.iteration.util.IterateExampleData;
-import org.apache.flink.streaming.examples.ml.util.IncrementalLearningSkeletonData;
import org.apache.flink.streaming.examples.twitter.util.TwitterExampleData;
import org.apache.flink.streaming.examples.windowing.util.SessionWindowingData;
import org.apache.flink.streaming.test.examples.join.WindowJoinData;
@@ -107,14 +106,6 @@ public class StreamingExamplesITCase extends AbstractTestBase {
}
@Test
- public void testIncrementalLearningSkeleton() throws Exception {
- final String resultPath = getTempDirPath("result");
- org.apache.flink.streaming.examples.ml.IncrementalLearningSkeleton.main(
- new String[] {"--output", resultPath});
- compareResultsByLinesInMemory(IncrementalLearningSkeletonData.RESULTS, resultPath);
- }
-
- @Test
public void testTwitterStream() throws Exception {
final String resultPath = getTempDirPath("result");
org.apache.flink.streaming.examples.twitter.TwitterExample.main(
diff --git a/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala b/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
index 7407294..de00b95 100644
--- a/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
+++ b/flink-examples/flink-examples-streaming/src/test/scala/org/apache/flink/streaming/scala/examples/StreamingExamplesITCase.scala
@@ -25,13 +25,11 @@ import org.apache.flink.core.fs.FileSystem.WriteMode
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.examples.iteration.util.IterateExampleData
-import org.apache.flink.streaming.examples.ml.util.IncrementalLearningSkeletonData
import org.apache.flink.streaming.examples.twitter.util.TwitterExampleData
import org.apache.flink.streaming.examples.windowing.util.SessionWindowingData
import org.apache.flink.streaming.scala.examples.iteration.IterateExample
import org.apache.flink.streaming.scala.examples.join.WindowJoin
import org.apache.flink.streaming.scala.examples.join.WindowJoin.{Grade, Salary}
-import org.apache.flink.streaming.scala.examples.ml.IncrementalLearningSkeleton
import org.apache.flink.streaming.scala.examples.twitter.TwitterExample
import org.apache.flink.streaming.scala.examples.windowing.{SessionWindowing, WindowWordCount}
import org.apache.flink.streaming.scala.examples.wordcount.WordCount
@@ -96,13 +94,6 @@ class StreamingExamplesITCase extends AbstractTestBase {
}
@Test
- def testIncrementalLearningSkeleton(): Unit = {
- val resultPath = getTempDirPath("result")
- IncrementalLearningSkeleton.main(Array("--output", resultPath))
- TestBaseUtils.compareResultsByLinesInMemory(IncrementalLearningSkeletonData.RESULTS, resultPath)
- }
-
- @Test
def testTwitterExample(): Unit = {
val resultPath = getTempDirPath("result")
TwitterExample.main(Array("--output", resultPath))
[flink] 02/05: [FLINK-21976] Remove fink-ml-* related configs from
pom.xml
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit a65b3ed334aa2abf3c7b24d60d1a3395a8d1835e
Author: Dong Lin <li...@gmail.com>
AuthorDate: Sun Mar 28 14:39:44 2021 +0800
[FLINK-21976] Remove fink-ml-* related configs from pom.xml
This closes #15405.
---
flink-dist/pom.xml | 7 -------
flink-dist/src/main/assemblies/opt.xml | 8 --------
flink-python/pyflink/pyflink_gateway_server.py | 2 --
pom.xml | 1 -
4 files changed, 18 deletions(-)
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index d0bed7f..8405e72 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -406,13 +406,6 @@ under the License.
<artifactId>flink-shaded-netty-tcnative-dynamic</artifactId>
<scope>provided</scope>
</dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-ml-uber_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
<!-- end optional Flink libraries -->
<dependency>
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index 74ea18d..23c197b 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -119,14 +119,6 @@
<fileMode>0644</fileMode>
</file>
- <!-- ML (API & LIB) -->
- <file>
- <source>../flink-ml-parent/flink-ml-uber/target/flink-ml-uber_${scala.binary.version}-${project.version}.jar</source>
- <outputDirectory>opt</outputDirectory>
- <destName>flink-ml_${scala.binary.version}-${project.version}.jar</destName>
- <fileMode>0644</fileMode>
- </file>
-
<file>
<source>target/temporary/flink-shaded-zookeeper-3-${zookeeper.optional.version}-${flink.shaded.version}.jar</source>
<outputDirectory>opt</outputDirectory>
diff --git a/flink-python/pyflink/pyflink_gateway_server.py b/flink-python/pyflink/pyflink_gateway_server.py
index ac8de68..640172f 100644
--- a/flink-python/pyflink/pyflink_gateway_server.py
+++ b/flink-python/pyflink/pyflink_gateway_server.py
@@ -161,8 +161,6 @@ def construct_test_classpath():
"flink-formats/flink-avro/target/flink-avro*.jar",
"flink-formats/flink-avro/target/avro*.jar",
"flink-formats/flink-json/target/flink-json*.jar",
- "flink-ml-parent/flink-ml-api/target/flink-ml-api*.jar",
- "flink-ml-parent/flink-ml-lib/target/flink-ml-lib*.jar",
"flink-python/target/artifacts/testDataStream.jar",
]
test_jars = []
diff --git a/pom.xml b/pom.xml
index b1d5bc8..5f97f70 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,7 +84,6 @@ under the License.
<module>flink-fs-tests</module>
<module>flink-docs</module>
<module>flink-python</module>
- <module>flink-ml-parent</module>
<module>flink-walkthroughs</module>
<module>flink-kubernetes</module>
<module>flink-external-resources</module>
[flink] 01/05: [FLINK-21976] Remove all files under flink-ml-parent
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 60b3895db28c2270f6e21a02e14b0834557be101
Author: Dong Lin <li...@gmail.com>
AuthorDate: Thu Mar 25 19:43:13 2021 +0800
[FLINK-21976] Remove all files under flink-ml-parent
This closes #15405.
---
flink-ml-parent/flink-ml-api/pom.xml | 47 --
.../org/apache/flink/ml/api/core/Estimator.java | 47 --
.../java/org/apache/flink/ml/api/core/Model.java | 37 --
.../org/apache/flink/ml/api/core/Pipeline.java | 259 ---------
.../apache/flink/ml/api/core/PipelineStage.java | 46 --
.../org/apache/flink/ml/api/core/Transformer.java | 42 --
.../apache/flink/ml/api/misc/param/ParamInfo.java | 151 ------
.../flink/ml/api/misc/param/ParamInfoFactory.java | 134 -----
.../flink/ml/api/misc/param/ParamValidator.java | 39 --
.../org/apache/flink/ml/api/misc/param/Params.java | 277 ----------
.../apache/flink/ml/api/misc/param/WithParams.java | 60 ---
.../flink/ml/util/param/ExtractParamInfosUtil.java | 71 ---
.../org/apache/flink/ml/api/core/PipelineTest.java | 167 ------
.../org/apache/flink/ml/api/misc/ParamsTest.java | 179 -------
.../ml/util/param/ExtractParamInfosUtilTest.java | 109 ----
flink-ml-parent/flink-ml-lib/pom.xml | 86 ---
.../org/apache/flink/ml/common/MLEnvironment.java | 151 ------
.../flink/ml/common/MLEnvironmentFactory.java | 116 -----
.../org/apache/flink/ml/common/linalg/BLAS.java | 234 ---------
.../apache/flink/ml/common/linalg/DenseMatrix.java | 577 ---------------------
.../apache/flink/ml/common/linalg/DenseVector.java | 379 --------------
.../apache/flink/ml/common/linalg/MatVecOp.java | 307 -----------
.../flink/ml/common/linalg/SparseVector.java | 574 --------------------
.../org/apache/flink/ml/common/linalg/Vector.java | 89 ----
.../flink/ml/common/linalg/VectorIterator.java | 73 ---
.../apache/flink/ml/common/linalg/VectorUtil.java | 240 ---------
.../org/apache/flink/ml/common/mapper/Mapper.java | 79 ---
.../flink/ml/common/mapper/MapperAdapter.java | 46 --
.../apache/flink/ml/common/mapper/ModelMapper.java | 66 ---
.../flink/ml/common/mapper/ModelMapperAdapter.java | 62 ---
.../common/model/BroadcastVariableModelSource.java | 47 --
.../apache/flink/ml/common/model/ModelSource.java | 40 --
.../flink/ml/common/model/RowsModelSource.java | 46 --
.../basicstatistic/MultivariateGaussian.java | 137 -----
.../ml/common/utils/DataSetConversionUtil.java | 167 ------
.../ml/common/utils/DataStreamConversionUtil.java | 167 ------
.../flink/ml/common/utils/OutputColsHelper.java | 211 --------
.../apache/flink/ml/common/utils/TableUtil.java | 439 ----------------
.../apache/flink/ml/common/utils/VectorTypes.java | 43 --
.../org/apache/flink/ml/operator/AlgoOperator.java | 186 -------
.../flink/ml/operator/batch/BatchOperator.java | 113 ----
.../operator/batch/source/TableSourceBatchOp.java | 40 --
.../flink/ml/operator/stream/StreamOperator.java | 114 ----
.../stream/source/TableSourceStreamOp.java | 40 --
.../flink/ml/params/shared/HasMLEnvironmentId.java | 43 --
.../ml/params/shared/colname/HasOutputCol.java | 48 --
.../shared/colname/HasOutputColDefaultAsNull.java | 49 --
.../ml/params/shared/colname/HasOutputCols.java | 48 --
.../shared/colname/HasOutputColsDefaultAsNull.java | 49 --
.../ml/params/shared/colname/HasPredictionCol.java | 42 --
.../shared/colname/HasPredictionDetailCol.java | 47 --
.../ml/params/shared/colname/HasReservedCols.java | 45 --
.../ml/params/shared/colname/HasSelectedCol.java | 48 --
.../colname/HasSelectedColDefaultAsNull.java | 49 --
.../ml/params/shared/colname/HasSelectedCols.java | 48 --
.../colname/HasSelectedColsDefaultAsNull.java | 49 --
.../apache/flink/ml/pipeline/EstimatorBase.java | 103 ----
.../org/apache/flink/ml/pipeline/ModelBase.java | 68 ---
.../flink/ml/pipeline/PipelineStageBase.java | 72 ---
.../apache/flink/ml/pipeline/TransformerBase.java | 99 ----
.../apache/flink/ml/common/MLEnvironmentTest.java | 83 ---
.../apache/flink/ml/common/linalg/BLASTest.java | 186 -------
.../flink/ml/common/linalg/DenseMatrixTest.java | 195 -------
.../flink/ml/common/linalg/DenseVectorTest.java | 158 ------
.../flink/ml/common/linalg/MatVecOpTest.java | 103 ----
.../flink/ml/common/linalg/SparseVectorTest.java | 232 ---------
.../flink/ml/common/linalg/VectorUtilTest.java | 76 ---
.../basicstatistic/MultivariateGaussianTest.java | 72 ---
.../ml/common/utils/DataSetConversionUtilTest.java | 200 -------
.../common/utils/DataStreamConversionUtilTest.java | 208 --------
.../ml/common/utils/OutputColsHelperTest.java | 249 ---------
.../flink/ml/common/utils/TableUtilTest.java | 200 -------
.../flink/ml/common/utils/VectorTypesTest.java | 78 ---
.../flink/ml/pipeline/EstimatorBaseTest.java | 89 ----
.../flink/ml/pipeline/PipelineStageTestBase.java | 70 ---
.../flink/ml/pipeline/TransformerBaseTest.java | 88 ----
.../ml/pipeline/UserDefinedPipelineStages.java | 58 ---
flink-ml-parent/flink-ml-uber/pom.xml | 76 ---
.../src/main/resources/META-INF/NOTICE | 10 -
.../main/resources/META-INF/licenses/LICENSE.core | 49 --
flink-ml-parent/pom.xml | 42 --
81 files changed, 9963 deletions(-)
diff --git a/flink-ml-parent/flink-ml-api/pom.xml b/flink-ml-parent/flink-ml-api/pom.xml
deleted file mode 100644
index 83df953..0000000
--- a/flink-ml-parent/flink-ml-api/pom.xml
+++ /dev/null
@@ -1,47 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-ml-parent</artifactId>
- <version>1.13-SNAPSHOT</version>
- </parent>
-
- <artifactId>flink-ml-api</artifactId>
- <name>Flink : ML : API</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-api-java</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-shaded-jackson</artifactId>
- <scope>provided</scope>
- </dependency>
- </dependencies>
-</project>
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Estimator.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Estimator.java
deleted file mode 100644
index 24c8349..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Estimator.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.core;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-
-/**
- * Estimators are {@link PipelineStage}s responsible for training and generating machine learning
- * models.
- *
- * <p>The implementations are expected to take an input table as training samples and generate a
- * {@link Model} which fits these samples.
- *
- * @param <E> class type of the Estimator implementation itself, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams}.
- * @param <M> class type of the {@link Model} this Estimator produces.
- */
-@PublicEvolving
-public interface Estimator<E extends Estimator<E, M>, M extends Model<M>> extends PipelineStage<E> {
-
- /**
- * Train and produce a {@link Model} which fits the records in the given {@link Table}.
- *
- * @param tEnv the table environment to which the input table is bound.
- * @param input the table with records to train the Model.
- * @return a model trained to fit on the given Table.
- */
- M fit(TableEnvironment tEnv, Table input);
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Model.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Model.java
deleted file mode 100644
index 6f15bc5..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Model.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.core;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.table.api.Table;
-
-/**
- * A model is an ordinary {@link Transformer} except how it is created. While ordinary transformers
- * are defined by specifying the parameters directly, a model is usually generated by an {@link
- * Estimator} when {@link Estimator#fit(org.apache.flink.table.api.TableEnvironment, Table)} is
- * invoked.
- *
- * <p>We separate Model from {@link Transformer} in order to support potential model specific logic
- * such as linking a Model to the {@link Estimator} from which the model was generated.
- *
- * @param <M> The class type of the Model implementation itself, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams}
- */
-@PublicEvolving
-public interface Model<M extends Model<M>> extends Transformer<M> {}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Pipeline.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Pipeline.java
deleted file mode 100644
index c5a56a5..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Pipeline.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.core;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-import org.apache.flink.util.InstantiationUtil;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * A pipeline is a linear workflow which chains {@link Estimator}s and {@link Transformer}s to
- * execute an algorithm.
- *
- * <p>A pipeline itself can either act as an Estimator or a Transformer, depending on the stages it
- * includes. More specifically:
- *
- * <ul>
- * <li>If a Pipeline has an {@link Estimator}, one needs to call {@link
- * Pipeline#fit(TableEnvironment, Table)} before use the pipeline as a {@link Transformer} .
- * In this case the Pipeline is an {@link Estimator} and can produce a Pipeline as a {@link
- * Model}.
- * <li>If a Pipeline has no {@link Estimator}, it is a {@link Transformer} and can be applied to a
- * Table directly. In this case, {@link Pipeline#fit(TableEnvironment, Table)} will simply
- * return the pipeline itself.
- * </ul>
- *
- * <p>In addition, a pipeline can also be used as a {@link PipelineStage} in another pipeline, just
- * like an ordinary {@link Estimator} or {@link Transformer} as describe above.
- */
-@PublicEvolving
-public final class Pipeline
- implements Estimator<Pipeline, Pipeline>, Transformer<Pipeline>, Model<Pipeline> {
- private static final long serialVersionUID = 1L;
- private final List<PipelineStage> stages = new ArrayList<>();
- private final Params params = new Params();
-
- private int lastEstimatorIndex = -1;
-
- public Pipeline() {}
-
- public Pipeline(String pipelineJson) {
- this.loadJson(pipelineJson);
- }
-
- public Pipeline(List<PipelineStage> stages) {
- for (PipelineStage s : stages) {
- appendStage(s);
- }
- }
-
- // is the stage a simple Estimator or pipeline with Estimator
- private static boolean isStageNeedFit(PipelineStage stage) {
- return (stage instanceof Pipeline && ((Pipeline) stage).needFit())
- || (!(stage instanceof Pipeline) && stage instanceof Estimator);
- }
-
- /**
- * Appends a PipelineStage to the tail of this pipeline. Pipeline is editable only via this
- * method. The PipelineStage must be Estimator, Transformer, Model or Pipeline.
- *
- * @param stage the stage to be appended
- */
- public Pipeline appendStage(PipelineStage stage) {
- if (isStageNeedFit(stage)) {
- lastEstimatorIndex = stages.size();
- } else if (!(stage instanceof Transformer)) {
- throw new RuntimeException(
- "All PipelineStages should be Estimator or Transformer, got:"
- + stage.getClass().getSimpleName());
- }
- stages.add(stage);
- return this;
- }
-
- /**
- * Returns a list of all stages in this pipeline in order, the list is immutable.
- *
- * @return an immutable list of all stages in this pipeline in order.
- */
- public List<PipelineStage> getStages() {
- return Collections.unmodifiableList(stages);
- }
-
- /**
- * Check whether the pipeline acts as an {@link Estimator} or not. When the return value is
- * true, that means this pipeline contains an {@link Estimator} and thus users must invoke
- * {@link #fit(TableEnvironment, Table)} before they can use this pipeline as a {@link
- * Transformer}. Otherwise, the pipeline can be used as a {@link Transformer} directly.
- *
- * @return {@code true} if this pipeline has an Estimator, {@code false} otherwise
- */
- public boolean needFit() {
- return this.getIndexOfLastEstimator() >= 0;
- }
-
- public Params getParams() {
- return params;
- }
-
- // find the last Estimator or Pipeline that needs fit in stages, -1 stand for no Estimator in
- // Pipeline
- private int getIndexOfLastEstimator() {
- return lastEstimatorIndex;
- }
-
- /**
- * Train the pipeline to fit on the records in the given {@link Table}.
- *
- * <p>This method go through all the {@link PipelineStage}s in order and does the following on
- * each stage until the last {@link Estimator}(inclusive).
- *
- * <ul>
- * <li>If a stage is an {@link Estimator}, invoke {@link Estimator#fit(TableEnvironment,
- * Table)} with the input table to generate a {@link Model}, transform the the input table
- * with the generated {@link Model} to get a result table, then pass the result table to
- * the next stage as input.
- * <li>If a stage is a {@link Transformer}, invoke {@link
- * Transformer#transform(TableEnvironment, Table)} on the input table to get a result
- * table, and pass the result table to the next stage as input.
- * </ul>
- *
- * <p>After all the {@link Estimator}s are trained to fit their input tables, a new pipeline
- * will be created with the same stages in this pipeline, except that all the Estimators in the
- * new pipeline are replaced with their corresponding Models generated in the above process.
- *
- * <p>If there is no {@link Estimator} in the pipeline, the method returns a copy of this
- * pipeline.
- *
- * @param tEnv the table environment to which the input table is bound.
- * @param input the table with records to train the Pipeline.
- * @return a pipeline with same stages as this Pipeline except all Estimators replaced with
- * their corresponding Models.
- */
- @Override
- public Pipeline fit(TableEnvironment tEnv, Table input) {
- List<PipelineStage> transformStages = new ArrayList<>(stages.size());
- int lastEstimatorIdx = getIndexOfLastEstimator();
- for (int i = 0; i < stages.size(); i++) {
- PipelineStage s = stages.get(i);
- if (i <= lastEstimatorIdx) {
- Transformer t;
- boolean needFit = isStageNeedFit(s);
- if (needFit) {
- t = ((Estimator) s).fit(tEnv, input);
- } else {
- // stage is Transformer, guaranteed in appendStage() method
- t = (Transformer) s;
- }
- transformStages.add(t);
- input = t.transform(tEnv, input);
- } else {
- transformStages.add(s);
- }
- }
- return new Pipeline(transformStages);
- }
-
- /**
- * Generate a result table by applying all the stages in this pipeline to the input table in
- * order.
- *
- * @param tEnv the table environment to which the input table is bound.
- * @param input the table to be transformed
- * @return a result table with all the stages applied to the input tables in order.
- */
- @Override
- public Table transform(TableEnvironment tEnv, Table input) {
- if (needFit()) {
- throw new RuntimeException("Pipeline contains Estimator, need to fit first.");
- }
- for (PipelineStage s : stages) {
- input = ((Transformer) s).transform(tEnv, input);
- }
- return input;
- }
-
- @Override
- public String toJson() {
- ObjectMapper mapper = new ObjectMapper();
-
- List<Map<String, String>> stageJsons = new ArrayList<>();
- for (PipelineStage s : getStages()) {
- Map<String, String> stageMap = new HashMap<>();
- stageMap.put("stageClassName", s.getClass().getTypeName());
- stageMap.put("stageJson", s.toJson());
- stageJsons.add(stageMap);
- }
-
- try {
- return mapper.writeValueAsString(stageJsons);
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Failed to serialize pipeline", e);
- }
- }
-
- @Override
- @SuppressWarnings("unchecked")
- public void loadJson(String json) {
- ObjectMapper mapper = new ObjectMapper();
- List<Map<String, String>> stageJsons;
- try {
- stageJsons = mapper.readValue(json, List.class);
- } catch (IOException e) {
- throw new RuntimeException("Failed to deserialize pipeline json:" + json, e);
- }
- for (Map<String, String> stageMap : stageJsons) {
- appendStage(restoreInnerStage(stageMap));
- }
- }
-
- private PipelineStage<?> restoreInnerStage(Map<String, String> stageMap) {
- String className = stageMap.get("stageClassName");
- Class<?> clz;
- try {
- clz = Class.forName(className);
- } catch (ClassNotFoundException e) {
- throw new RuntimeException("PipelineStage class " + className + " not exists", e);
- }
- InstantiationUtil.checkForInstantiation(clz);
-
- PipelineStage<?> s;
- try {
- s = (PipelineStage<?>) clz.newInstance();
- } catch (Exception e) {
- throw new RuntimeException("Class is instantiable but failed to new an instance", e);
- }
-
- String stageJson = stageMap.get("stageJson");
- s.loadJson(stageJson);
- return s;
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/PipelineStage.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/PipelineStage.java
deleted file mode 100644
index 0a3dd23..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/PipelineStage.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.core;
-
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-import java.io.Serializable;
-
-/**
- * Base class for a stage in a pipeline. The interface is only a concept, and does not have any
- * actual functionality. Its subclasses must be either Estimator or Transformer. No other classes
- * should inherit this interface directly.
- *
- * <p>Each pipeline stage is with parameters, and requires a public empty constructor for
- * restoration in Pipeline.
- *
- * @param <T> The class type of the PipelineStage implementation itself, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams}
- * @see WithParams
- */
-interface PipelineStage<T extends PipelineStage<T>> extends WithParams<T>, Serializable {
-
- default String toJson() {
- return getParams().toJson();
- }
-
- default void loadJson(String json) {
- getParams().loadJson(json);
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Transformer.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Transformer.java
deleted file mode 100644
index f9a152c..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/core/Transformer.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.core;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-
-/**
- * A transformer is a {@link PipelineStage} that transforms an input {@link Table} to a result
- * {@link Table}.
- *
- * @param <T> The class type of the Transformer implementation itself, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams}
- */
-@PublicEvolving
-public interface Transformer<T extends Transformer<T>> extends PipelineStage<T> {
- /**
- * Applies the transformer on the input table, and returns the result table.
- *
- * @param tEnv the table environment to which the input table is bound.
- * @param input the table to be transformed
- * @return the transformed table
- */
- Table transform(TableEnvironment tEnv, Table input);
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfo.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfo.java
deleted file mode 100644
index b0f7ce9..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfo.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.misc.param;
-
-import org.apache.flink.annotation.PublicEvolving;
-import org.apache.flink.util.Preconditions;
-
-/**
- * Definition of a parameter, including name, type, default value, validator and so on.
- *
- * <p>A parameter can either be optional or non-optional.
- *
- * <ul>
- * <li>A non-optional parameter should not have a default value. Instead, its value must be
- * provided by the users.
- * <li>An optional parameter may or may not have a default value.
- * </ul>
- *
- * <p>Please see {@link Params#get(ParamInfo)} and {@link Params#contains(ParamInfo)} for more
- * details about the behavior.
- *
- * <p>A parameter may have aliases in addition to the parameter name for convenience and
- * compatibility purposes. One should not set values for both parameter name and an alias. One and
- * only one value should be set either under the parameter name or one of the alias.
- *
- * @param <V> the type of the param value
- */
-@PublicEvolving
-public class ParamInfo<V> {
- private final String name;
- private final String[] alias;
- private final String description;
- private final boolean isOptional;
- private final boolean hasDefaultValue;
- private final V defaultValue;
- private final ParamValidator<V> validator;
- private final Class<V> valueClass;
-
- ParamInfo(
- String name,
- String[] alias,
- String description,
- boolean isOptional,
- boolean hasDefaultValue,
- V defaultValue,
- ParamValidator<V> validator,
- Class<V> valueClass) {
- this.name = name;
- this.alias = alias;
- this.description = description;
- this.isOptional = isOptional;
- this.hasDefaultValue = hasDefaultValue;
- this.defaultValue = defaultValue;
- this.validator = validator;
- this.valueClass = valueClass;
- }
-
- /**
- * Returns the name of the parameter. The name must be unique in the stage the ParamInfo belongs
- * to.
- *
- * @return the name of the parameter
- */
- public String getName() {
- return name;
- }
-
- /**
- * Returns the aliases of the parameter. The alias will be an empty string array by default.
- *
- * @return the aliases of the parameter
- */
- public String[] getAlias() {
- Preconditions.checkNotNull(alias);
- return alias;
- }
-
- /**
- * Returns the description of the parameter.
- *
- * @return the description of the parameter
- */
- public String getDescription() {
- return description;
- }
-
- /**
- * Returns whether the parameter is optional.
- *
- * @return {@code true} if the param is optional, {@code false} otherwise
- */
- public boolean isOptional() {
- return isOptional;
- }
-
- /**
- * Returns whether the parameter has a default value. Since {@code null} may also be a valid
- * default value of a parameter, the return of getDefaultValue may be {@code null} even when
- * this method returns true.
- *
- * @return {@code true} if the param is has a default value(even if it's a {@code null}), {@code
- * false} otherwise
- */
- public boolean hasDefaultValue() {
- return hasDefaultValue;
- }
-
- /**
- * Returns the default value of the parameter. The default value should be defined whenever
- * possible. The default value can be a {@code null} even if hasDefaultValue returns true.
- *
- * @return the default value of the param, {@code null} if not defined
- */
- public V getDefaultValue() {
- return defaultValue;
- }
-
- /**
- * Returns the validator to validate the value of the parameter.
- *
- * @return the validator to validate the value of the parameter.
- */
- public ParamValidator<V> getValidator() {
- return validator;
- }
-
- /**
- * Returns the class of the param value. It's usually needed in serialization.
- *
- * @return the class of the param value
- */
- public Class<V> getValueClass() {
- return valueClass;
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfoFactory.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfoFactory.java
deleted file mode 100644
index 5199a5d..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamInfoFactory.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.misc.param;
-
-/** Factory to create ParamInfo, all ParamInfos should be created via this class. */
-public class ParamInfoFactory {
- /**
- * Returns a ParamInfoBuilder to configure and build a new ParamInfo.
- *
- * @param name name of the new ParamInfo
- * @param valueClass value class of the new ParamInfo
- * @param <V> value type of the new ParamInfo
- * @return a ParamInfoBuilder
- */
- public static <V> ParamInfoBuilder<V> createParamInfo(String name, Class<V> valueClass) {
- return new ParamInfoBuilder<>(name, valueClass);
- }
-
- /**
- * Builder to build a new ParamInfo. Builder is created by ParamInfoFactory with name and
- * valueClass set.
- *
- * @param <V> value type of the new ParamInfo
- */
- public static class ParamInfoBuilder<V> {
- private String name;
- private String[] alias = new String[0];
- private String description;
- private boolean isOptional = true;
- private boolean hasDefaultValue = false;
- private V defaultValue;
- private ParamValidator<V> validator;
- private Class<V> valueClass;
-
- ParamInfoBuilder(String name, Class<V> valueClass) {
- this.name = name;
- this.valueClass = valueClass;
- }
-
- /**
- * Sets the aliases of the parameter.
- *
- * @return the builder itself
- */
- public ParamInfoBuilder<V> setAlias(String[] alias) {
- this.alias = alias;
- return this;
- }
-
- /**
- * Sets the description of the parameter.
- *
- * @return the builder itself
- */
- public ParamInfoBuilder<V> setDescription(String description) {
- this.description = description;
- return this;
- }
-
- /**
- * Sets the flag indicating the parameter is optional. The parameter is optional by default.
- *
- * @return the builder itself
- */
- public ParamInfoBuilder<V> setOptional() {
- this.isOptional = true;
- return this;
- }
-
- /**
- * Sets the flag indicating the parameter is required.
- *
- * @return the builder itself
- */
- public ParamInfoBuilder<V> setRequired() {
- this.isOptional = false;
- return this;
- }
-
- /**
- * Sets the flag indicating the parameter has default value, and sets the default value.
- *
- * @return the builder itself
- */
- public ParamInfoBuilder<V> setHasDefaultValue(V defaultValue) {
- this.hasDefaultValue = true;
- this.defaultValue = defaultValue;
- return this;
- }
-
- /**
- * Sets the validator to validate the parameter value set by users.
- *
- * @return the builder itself
- */
- public ParamInfoBuilder<V> setValidator(ParamValidator<V> validator) {
- this.validator = validator;
- return this;
- }
-
- /**
- * Builds the defined ParamInfo and returns it. The ParamInfo will be immutable.
- *
- * @return the defined ParamInfo
- */
- public ParamInfo<V> build() {
- return new ParamInfo<>(
- name,
- alias,
- description,
- isOptional,
- hasDefaultValue,
- defaultValue,
- validator,
- valueClass);
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamValidator.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamValidator.java
deleted file mode 100644
index d41c709..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/ParamValidator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.misc.param;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-import java.io.Serializable;
-
-/**
- * An interface used by {@link ParamInfo} to do validation when a parameter value is set.
- *
- * @param <V> the type of the value to validate
- */
-@PublicEvolving
-public interface ParamValidator<V> extends Serializable {
- /**
- * Validates a parameter value.
- *
- * @param value value to validate
- * @return {@code true} if the value is valid, {@code false} otherwise
- */
- boolean validate(V value);
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/Params.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/Params.java
deleted file mode 100644
index e173a66..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/Params.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.misc.param;
-
-import org.apache.flink.annotation.PublicEvolving;
-
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.core.JsonProcessingException;
-import org.apache.flink.shaded.jackson2.com.fasterxml.jackson.databind.ObjectMapper;
-
-import java.io.IOException;
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-/**
- * The map-like container class for parameter. This class is provided to unify the interaction with
- * parameters.
- */
-@PublicEvolving
-public class Params implements Serializable, Cloneable {
- private static final long serialVersionUID = 1L;
-
- /**
- * A mapping from param name to its value.
- *
- * <p>The value is stored in map using json format.
- */
- private final Map<String, String> params;
-
- private transient ObjectMapper mapper;
-
- public Params() {
- this.params = new HashMap<>();
- }
-
- /**
- * Return the number of params.
- *
- * @return Return the number of params.
- */
- public int size() {
- return params.size();
- }
-
- /** Removes all of the params. The params will be empty after this call returns. */
- public void clear() {
- params.clear();
- }
-
- /**
- * Returns <tt>true</tt> if this params contains no mappings.
- *
- * @return <tt>true</tt> if this map contains no mappings
- */
- public boolean isEmpty() {
- return params.isEmpty();
- }
-
- /**
- * Returns the value of the specific parameter, or default value defined in the {@code info} if
- * this Params doesn't have a value set for the parameter. An exception will be thrown in the
- * following cases because no value could be found for the specified parameter.
- *
- * <ul>
- * <li>Non-optional parameter: no value is defined in this params for a non-optional
- * parameter.
- * <li>Optional parameter: no value is defined in this params and no default value is defined.
- * </ul>
- *
- * @param info the info of the specific parameter, usually with default value
- * @param <V> the type of the specific parameter
- * @return the value of the specific parameter, or default value defined in the {@code info} if
- * this Params doesn't contain the parameter
- * @throws IllegalArgumentException if no value can be found for specified parameter
- */
- public <V> V get(ParamInfo<V> info) {
- String value = null;
- String usedParamName = null;
- for (String nameOrAlias : getParamNameAndAlias(info)) {
- if (params.containsKey(nameOrAlias)) {
- if (usedParamName != null) {
- throw new IllegalArgumentException(
- String.format(
- "Duplicate parameters of %s and %s",
- usedParamName, nameOrAlias));
- }
- usedParamName = nameOrAlias;
- value = params.get(nameOrAlias);
- }
- }
-
- if (usedParamName != null) {
- // The param value was set by the user.
- return valueFromJson(value, info.getValueClass());
- } else {
- // The param value was not set by the user.
- if (!info.isOptional()) {
- throw new IllegalArgumentException(
- "Missing non-optional parameter " + info.getName());
- } else if (!info.hasDefaultValue()) {
- throw new IllegalArgumentException(
- "Cannot find default value for optional parameter " + info.getName());
- }
- return info.getDefaultValue();
- }
- }
-
- /**
- * Set the value of the specific parameter.
- *
- * @param info the info of the specific parameter to set.
- * @param value the value to be set to the specific parameter.
- * @param <V> the type of the specific parameter.
- * @return the previous value of the specific parameter, or null if this Params didn't contain
- * the parameter before
- * @throws RuntimeException if the {@code info} has a validator and the {@code value} is
- * evaluated as illegal by the validator
- */
- public <V> Params set(ParamInfo<V> info, V value) {
- if (info.getValidator() != null && !info.getValidator().validate(value)) {
- throw new RuntimeException(
- "Setting " + info.getName() + " as a invalid value:" + value);
- }
- params.put(info.getName(), valueToJson(value));
- return this;
- }
-
- /**
- * Removes the specific parameter from this Params.
- *
- * @param info the info of the specific parameter to remove
- * @param <V> the type of the specific parameter
- */
- public <V> void remove(ParamInfo<V> info) {
- params.remove(info.getName());
- for (String a : info.getAlias()) {
- params.remove(a);
- }
- }
-
- /**
- * Check whether this params has a value set for the given {@code info}.
- *
- * @return <tt>true</tt> if this params has a value set for the specified {@code info}, false
- * otherwise.
- */
- public <V> boolean contains(ParamInfo<V> info) {
- return params.containsKey(info.getName())
- || Arrays.stream(info.getAlias()).anyMatch(params::containsKey);
- }
-
- /**
- * Returns a json containing all parameters in this Params. The json should be human-readable if
- * possible.
- *
- * @return a json containing all parameters in this Params
- */
- public String toJson() {
- assertMapperInited();
- try {
- return mapper.writeValueAsString(params);
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Failed to serialize params to json", e);
- }
- }
-
- /**
- * Restores the parameters from the given json. The parameters should be exactly the same with
- * the one who was serialized to the input json after the restoration.
- *
- * @param json the json String to restore from
- */
- @SuppressWarnings("unchecked")
- public void loadJson(String json) {
- assertMapperInited();
- Map<String, String> params;
- try {
- params = mapper.readValue(json, Map.class);
- } catch (IOException e) {
- throw new RuntimeException("Failed to deserialize json:" + json, e);
- }
- this.params.putAll(params);
- }
-
- /**
- * Factory method for constructing params.
- *
- * @param json the json string to load
- * @return the {@code Params} loaded from the json string.
- */
- public static Params fromJson(String json) {
- Params params = new Params();
- params.loadJson(json);
- return params;
- }
-
- /**
- * Merge other params into this.
- *
- * @param otherParams other params
- * @return this
- */
- public Params merge(Params otherParams) {
- if (otherParams != null) {
- this.params.putAll(otherParams.params);
- }
- return this;
- }
-
- /**
- * Creates and returns a deep clone of this Params.
- *
- * @return a deep clone of this Params
- */
- @Override
- public Params clone() {
- Params newParams = new Params();
- newParams.params.putAll(this.params);
- return newParams;
- }
-
- private void assertMapperInited() {
- if (mapper == null) {
- mapper = new ObjectMapper();
- }
- }
-
- private String valueToJson(Object value) {
- assertMapperInited();
- try {
- if (value == null) {
- return null;
- }
- return mapper.writeValueAsString(value);
- } catch (JsonProcessingException e) {
- throw new RuntimeException("Failed to serialize to json:" + value, e);
- }
- }
-
- private <T> T valueFromJson(String json, Class<T> clazz) {
- assertMapperInited();
- try {
- if (json == null) {
- return null;
- }
- return mapper.readValue(json, clazz);
- } catch (IOException e) {
- throw new RuntimeException("Failed to deserialize json:" + json, e);
- }
- }
-
- private <V> List<String> getParamNameAndAlias(ParamInfo<V> info) {
- List<String> paramNames = new ArrayList<>(info.getAlias().length + 1);
- paramNames.add(info.getName());
- paramNames.addAll(Arrays.asList(info.getAlias()));
- return paramNames;
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/WithParams.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/WithParams.java
deleted file mode 100644
index dfbf614..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/api/misc/param/WithParams.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.misc.param;
-
-/**
- * Parameters are widely used in machine learning realm. This class defines a common interface to
- * interact with classes with parameters.
- *
- * @param <T> the actual type of this WithParams, as the return type of setter
- */
-public interface WithParams<T> {
- /**
- * Returns the all the parameters.
- *
- * @return all the parameters.
- */
- Params getParams();
-
- /**
- * Set the value of a specific parameter.
- *
- * @param info the info of the specific param to set
- * @param value the value to be set to the specific param
- * @param <V> the type of the specific param
- * @return the WithParams itself
- */
- @SuppressWarnings("unchecked")
- default <V> T set(ParamInfo<V> info, V value) {
- getParams().set(info, value);
- return (T) this;
- }
-
- /**
- * Returns the value of the specific param.
- *
- * @param info the info of the specific param, usually with default value
- * @param <V> the type of the specific param
- * @return the value of the specific param, or default value defined in the {@code info} if the
- * inner Params doesn't contains this param
- */
- default <V> V get(ParamInfo<V> info) {
- return getParams().get(info);
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/util/param/ExtractParamInfosUtil.java b/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/util/param/ExtractParamInfosUtil.java
deleted file mode 100644
index 5206c0a..0000000
--- a/flink-ml-parent/flink-ml-api/src/main/java/org/apache/flink/ml/util/param/ExtractParamInfosUtil.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.util.param;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.lang.reflect.Field;
-import java.util.ArrayList;
-import java.util.List;
-
-/** Utility to extract all ParamInfos defined in a WithParams, mainly used in persistence. */
-public final class ExtractParamInfosUtil {
- private static final Logger LOG = LoggerFactory.getLogger(ExtractParamInfosUtil.class);
-
- /**
- * Extracts all ParamInfos defined in the given WithParams, including those in its superclasses
- * and interfaces.
- *
- * @param s the WithParams to extract ParamInfos from
- * @return the list of all ParamInfos defined in s
- */
- public static List<ParamInfo> extractParamInfos(WithParams s) {
- return extractParamInfos(s, s.getClass());
- }
-
- private static List<ParamInfo> extractParamInfos(WithParams s, Class clz) {
- List<ParamInfo> result = new ArrayList<>();
- if (clz == null) {
- return result;
- }
-
- Field[] fields = clz.getDeclaredFields();
- for (Field f : fields) {
- f.setAccessible(true);
- if (ParamInfo.class.isAssignableFrom(f.getType())) {
- try {
- result.add((ParamInfo) f.get(s));
- } catch (IllegalAccessException e) {
- LOG.warn("Failed to extract param info {}, ignore it", f.getName(), e);
- }
- }
- }
-
- result.addAll(extractParamInfos(s, clz.getSuperclass()));
- for (Class c : clz.getInterfaces()) {
- result.addAll(extractParamInfos(s, c));
- }
-
- return result;
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/core/PipelineTest.java b/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/core/PipelineTest.java
deleted file mode 100644
index 87ec13a..0000000
--- a/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/core/PipelineTest.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.core;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-/** Tests the behavior of {@link Pipeline}. */
-public class PipelineTest {
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testPipelineBehavior() {
- Pipeline pipeline = new Pipeline();
- pipeline.appendStage(new MockTransformer("a"));
- pipeline.appendStage(new MockEstimator("b"));
- pipeline.appendStage(new MockEstimator("c"));
- pipeline.appendStage(new MockTransformer("d"));
- assert describePipeline(pipeline).equals("a_b_c_d");
-
- Pipeline pipelineModel = pipeline.fit(null, null);
- assert describePipeline(pipelineModel).equals("a_mb_mc_d");
-
- thrown.expect(RuntimeException.class);
- thrown.expectMessage("Pipeline contains Estimator, need to fit first.");
- pipeline.transform(null, null);
- }
-
- @Test
- public void testPipelineRestore() {
- Pipeline pipeline = new Pipeline();
- pipeline.appendStage(new MockTransformer("a"));
- pipeline.appendStage(new MockEstimator("b"));
- pipeline.appendStage(new MockEstimator("c"));
- pipeline.appendStage(new MockTransformer("d"));
- String pipelineJson = pipeline.toJson();
-
- Pipeline restoredPipeline = new Pipeline(pipelineJson);
- assert describePipeline(restoredPipeline).equals("a_b_c_d");
-
- Pipeline pipelineModel = pipeline.fit(null, null);
- String modelJson = pipelineModel.toJson();
-
- Pipeline restoredPipelineModel = new Pipeline(modelJson);
- assert describePipeline(restoredPipelineModel).equals("a_mb_mc_d");
- }
-
- private static String describePipeline(Pipeline p) {
- StringBuilder res = new StringBuilder();
- for (PipelineStage s : p.getStages()) {
- if (res.length() != 0) {
- res.append("_");
- }
- res.append(((SelfDescribe) s).describe());
- }
- return res.toString();
- }
-
- /** Interface to describe a class with a string, only for pipeline test. */
- private interface SelfDescribe {
- ParamInfo<String> DESCRIPTION =
- ParamInfoFactory.createParamInfo("description", String.class).build();
-
- String describe();
- }
-
- /** Mock estimator for pipeline test. */
- public static class MockEstimator implements Estimator<MockEstimator, MockModel>, SelfDescribe {
- private final Params params = new Params();
-
- public MockEstimator() {}
-
- MockEstimator(String description) {
- set(DESCRIPTION, description);
- }
-
- @Override
- public MockModel fit(TableEnvironment tEnv, Table input) {
- return new MockModel("m" + describe());
- }
-
- @Override
- public Params getParams() {
- return params;
- }
-
- @Override
- public String describe() {
- return get(DESCRIPTION);
- }
- }
-
- /** Mock transformer for pipeline test. */
- public static class MockTransformer implements Transformer<MockTransformer>, SelfDescribe {
- private final Params params = new Params();
-
- public MockTransformer() {}
-
- MockTransformer(String description) {
- set(DESCRIPTION, description);
- }
-
- @Override
- public Table transform(TableEnvironment tEnv, Table input) {
- return input;
- }
-
- @Override
- public Params getParams() {
- return params;
- }
-
- @Override
- public String describe() {
- return get(DESCRIPTION);
- }
- }
-
- /** Mock model for pipeline test. */
- public static class MockModel implements Model<MockModel>, SelfDescribe {
- private final Params params = new Params();
-
- public MockModel() {}
-
- MockModel(String description) {
- set(DESCRIPTION, description);
- }
-
- @Override
- public Table transform(TableEnvironment tEnv, Table input) {
- return input;
- }
-
- @Override
- public Params getParams() {
- return params;
- }
-
- @Override
- public String describe() {
- return get(DESCRIPTION);
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/misc/ParamsTest.java b/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/misc/ParamsTest.java
deleted file mode 100644
index 080e22b..0000000
--- a/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/api/misc/ParamsTest.java
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.api.misc;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.Params;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-/** Test for the behavior and validator of {@link Params}. */
-public class ParamsTest {
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testDefaultBehavior() {
- Params params = new Params();
-
- ParamInfo<String> optionalWithoutDefault =
- ParamInfoFactory.createParamInfo("a", String.class).build();
-
- // It should call params.contain to check when get the param in this case.
- thrown.expect(RuntimeException.class);
- thrown.expectMessage("Cannot find default value for optional parameter a");
- params.get(optionalWithoutDefault);
-
- ParamInfo<String> optionalWithDefault =
- ParamInfoFactory.createParamInfo("a", String.class)
- .setHasDefaultValue("def")
- .build();
- assert params.get(optionalWithDefault).equals("def");
-
- ParamInfo<String> requiredWithDefault =
- ParamInfoFactory.createParamInfo("a", String.class)
- .setRequired()
- .setHasDefaultValue("def")
- .build();
- assert params.get(requiredWithDefault).equals("def");
-
- ParamInfo<String> requiredWithoutDefault =
- ParamInfoFactory.createParamInfo("a", String.class).setRequired().build();
- thrown.expect(RuntimeException.class);
- thrown.expectMessage("a not exist which is not optional and don't have a default value");
- params.get(requiredWithoutDefault);
- }
-
- @Test
- public void testValidator() {
- Params params = new Params();
-
- ParamInfo<Integer> intParam =
- ParamInfoFactory.createParamInfo("a", Integer.class)
- .setValidator(i -> i > 0)
- .build();
- params.set(intParam, 1);
-
- thrown.expect(RuntimeException.class);
- thrown.expectMessage("Setting a as a invalid value:0");
- params.set(intParam, 0);
- }
-
- @Test
- public void getOptionalParam() {
- ParamInfo<String> key =
- ParamInfoFactory.createParamInfo("key", String.class)
- .setHasDefaultValue(null)
- .setDescription("")
- .build();
-
- Params params = new Params();
- Assert.assertNull(params.get(key));
-
- String val = "3";
- params.set(key, val);
- Assert.assertEquals(params.get(key), val);
-
- params.set(key, null);
- Assert.assertNull(params.get(key));
- }
-
- @Test
- public void getOptionalWithoutDefaultParam() {
- ParamInfo<String> key =
- ParamInfoFactory.createParamInfo("key", String.class)
- .setOptional()
- .setDescription("")
- .build();
- Params params = new Params();
-
- try {
- String val = params.get(key);
- Assert.fail("Should throw exception.");
- } catch (IllegalArgumentException ex) {
- Assert.assertTrue(
- ex.getMessage().startsWith("Cannot find default value for optional parameter"));
- }
-
- Assert.assertFalse(params.contains(key));
-
- String val = "3";
- params.set(key, val);
- Assert.assertEquals(params.get(key), val);
-
- Assert.assertTrue(params.contains(key));
-
- params.set(key, null);
- Assert.assertNull(params.get(key));
- }
-
- @Test
- public void getRequiredParam() {
- ParamInfo<String> labelWithRequired =
- ParamInfoFactory.createParamInfo("label", String.class)
- .setDescription("")
- .setRequired()
- .build();
- Params params = new Params();
- try {
- params.get(labelWithRequired);
- Assert.fail("failure");
- } catch (IllegalArgumentException ex) {
- Assert.assertTrue(ex.getMessage().startsWith("Missing non-optional parameter"));
- }
-
- params.set(labelWithRequired, null);
- Assert.assertNull(params.get(labelWithRequired));
-
- String val = "3";
- params.set(labelWithRequired, val);
- Assert.assertEquals(params.get(labelWithRequired), val);
- }
-
- @Test
- public void testGetAliasParam() {
- ParamInfo<String> predResultColName =
- ParamInfoFactory.createParamInfo("predResultColName", String.class)
- .setDescription("Column name of predicted result.")
- .setRequired()
- .setAlias(new String[] {"predColName", "outputColName"})
- .build();
-
- Params params = Params.fromJson("{\"predResultColName\":\"\\\"f0\\\"\"}");
-
- Assert.assertEquals("f0", params.get(predResultColName));
-
- params =
- Params.fromJson(
- "{\"predResultColName\":\"\\\"f0\\\"\", \"predColName\":\"\\\"f0\\\"\"}");
-
- try {
- params.get(predResultColName);
- Assert.fail("failure");
- } catch (IllegalArgumentException ex) {
- Assert.assertTrue(
- ex.getMessage()
- .startsWith(
- "Duplicate parameters of predResultColName and predColName"));
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/util/param/ExtractParamInfosUtilTest.java b/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/util/param/ExtractParamInfosUtilTest.java
deleted file mode 100644
index 1010bfb..0000000
--- a/flink-ml-parent/flink-ml-api/src/test/java/org/apache/flink/ml/util/param/ExtractParamInfosUtilTest.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.util.param;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-import org.junit.Test;
-
-import java.util.Arrays;
-import java.util.List;
-
-/** Test for {@link ExtractParamInfosUtil}. */
-public class ExtractParamInfosUtilTest {
-
- @Test
- public void testExtractParamInfos() {
- List<ParamInfo> noParamInfos =
- ExtractParamInfosUtil.extractParamInfos(new WithNoParamInfo());
- assert noParamInfos.isEmpty();
-
- List<ParamInfo> classParamInfos =
- ExtractParamInfosUtil.extractParamInfos(new WithTestParamInfo());
- assert classParamInfos.size() == 1 && classParamInfos.get(0).getName().equals("KSC");
-
- List<ParamInfo> allParamInfos =
- ExtractParamInfosUtil.extractParamInfos(new TestParamInfoWithInheritedParamInfos());
- String[] sortedCorrectParamNames = new String[] {"KCP", "KI", "KSC"};
- assert allParamInfos.size() == 3
- && Arrays.equals(
- sortedCorrectParamNames,
- allParamInfos.stream()
- .map(ParamInfo::getName)
- .sorted()
- .toArray(String[]::new));
- }
-
- /** Mock WithParams implementation with no ParamInfo. Only for test. */
- public static class WithNoParamInfo implements WithParams<WithNoParamInfo> {
-
- @Override
- public Params getParams() {
- return null;
- }
- }
-
- /**
- * Mock WithParams implementation with one ParamInfo. Only for test.
- *
- * @param <T> subclass of WithTestParamInfo
- */
- public static class WithTestParamInfo<T extends WithTestParamInfo> implements WithParams<T> {
- public static final ParamInfo<String> KSC =
- ParamInfoFactory.createParamInfo("KSC", String.class)
- .setDescription("key from super class")
- .build();
-
- @Override
- public Params getParams() {
- return null;
- }
- }
-
- /**
- * Mock interface extending WithParams with one ParamInfo. Only for test.
- *
- * @param <T> implementation class of InterfaceWithParamInfo
- */
- public interface InterfaceWithParamInfo<T extends InterfaceWithParamInfo>
- extends WithParams<T> {
- ParamInfo<String> KI =
- ParamInfoFactory.createParamInfo("KI", String.class)
- .setDescription("key from interface")
- .build();
- }
-
- /** Mock WithParams inheriting ParamInfos from superclass and interface. Only for test. */
- public static class TestParamInfoWithInheritedParamInfos
- extends WithTestParamInfo<TestParamInfoWithInheritedParamInfos>
- implements InterfaceWithParamInfo<TestParamInfoWithInheritedParamInfos> {
- private static final ParamInfo<String> KCP =
- ParamInfoFactory.createParamInfo("KCP", String.class)
- .setDescription("key in the class which is private")
- .build();
-
- @Override
- public Params getParams() {
- return null;
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/pom.xml b/flink-ml-parent/flink-ml-lib/pom.xml
deleted file mode 100644
index bd5102e..0000000
--- a/flink-ml-parent/flink-ml-lib/pom.xml
+++ /dev/null
@@ -1,86 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
-Licensed to the Apache Software Foundation (ASF) under one
-or more contributor license agreements. See the NOTICE file
-distributed with this work for additional information
-regarding copyright ownership. The ASF licenses this file
-to you under the Apache License, Version 2.0 (the
-"License"); you may not use this file except in compliance
-with the License. You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing,
-software distributed under the License is distributed on an
-"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-KIND, either express or implied. See the License for the
-specific language governing permissions and limitations
-under the License.
--->
-<project xmlns="http://maven.apache.org/POM/4.0.0"
- xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-ml-parent</artifactId>
- <version>1.13-SNAPSHOT</version>
- </parent>
-
- <artifactId>flink-ml-lib_${scala.binary.version}</artifactId>
- <name>Flink : ML : Lib</name>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-ml-api</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-api-java</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-api-java-bridge_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-table-planner_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
- <dependency>
- <groupId>com.github.fommil.netlib</groupId>
- <artifactId>core</artifactId>
- <version>1.1.2</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <!-- Because PyFlink uses it in tests -->
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java
deleted file mode 100644
index cff7ecd..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironment.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common;
-
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.bridge.java.BatchTableEnvironment;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-
-/**
- * The MLEnvironment stores the necessary context in Flink. Each MLEnvironment will be associated
- * with a unique ID. The operations associated with the same MLEnvironment ID will share the same
- * Flink job context.
- *
- * <p>Both MLEnvironment ID and MLEnvironment can only be retrieved from MLEnvironmentFactory.
- *
- * @see ExecutionEnvironment
- * @see StreamExecutionEnvironment
- * @see BatchTableEnvironment
- * @see StreamTableEnvironment
- */
-public class MLEnvironment {
- private ExecutionEnvironment env;
- private StreamExecutionEnvironment streamEnv;
- private BatchTableEnvironment batchTableEnv;
- private StreamTableEnvironment streamTableEnv;
-
- /** Construct with null that the class can load the environment in the `get` method. */
- public MLEnvironment() {
- this(null, null, null, null);
- }
-
- /**
- * Construct with the batch environment and the the batch table environment given by user.
- *
- * <p>The env can be null which will be loaded in the `get` method.
- *
- * @param batchEnv the ExecutionEnvironment
- * @param batchTableEnv the BatchTableEnvironment
- */
- public MLEnvironment(ExecutionEnvironment batchEnv, BatchTableEnvironment batchTableEnv) {
- this(batchEnv, batchTableEnv, null, null);
- }
-
- /**
- * Construct with the stream environment and the the stream table environment given by user.
- *
- * <p>The env can be null which will be loaded in the `get` method.
- *
- * @param streamEnv the StreamExecutionEnvironment
- * @param streamTableEnv the StreamTableEnvironment
- */
- public MLEnvironment(
- StreamExecutionEnvironment streamEnv, StreamTableEnvironment streamTableEnv) {
- this(null, null, streamEnv, streamTableEnv);
- }
-
- /**
- * Construct with env given by user.
- *
- * <p>The env can be null which will be loaded in the `get` method.
- *
- * @param batchEnv the ExecutionEnvironment
- * @param batchTableEnv the BatchTableEnvironment
- * @param streamEnv the StreamExecutionEnvironment
- * @param streamTableEnv the StreamTableEnvironment
- */
- public MLEnvironment(
- ExecutionEnvironment batchEnv,
- BatchTableEnvironment batchTableEnv,
- StreamExecutionEnvironment streamEnv,
- StreamTableEnvironment streamTableEnv) {
- this.env = batchEnv;
- this.batchTableEnv = batchTableEnv;
- this.streamEnv = streamEnv;
- this.streamTableEnv = streamTableEnv;
- }
-
- /**
- * Get the ExecutionEnvironment. if the ExecutionEnvironment has not been set, it initial the
- * ExecutionEnvironment with default Configuration.
- *
- * @return the batch {@link ExecutionEnvironment}
- */
- public ExecutionEnvironment getExecutionEnvironment() {
- if (null == env) {
- env = ExecutionEnvironment.getExecutionEnvironment();
- }
- return env;
- }
-
- /**
- * Get the StreamExecutionEnvironment. if the StreamExecutionEnvironment has not been set, it
- * initial the StreamExecutionEnvironment with default Configuration.
- *
- * @return the {@link StreamExecutionEnvironment}
- */
- public StreamExecutionEnvironment getStreamExecutionEnvironment() {
- if (null == streamEnv) {
- streamEnv = StreamExecutionEnvironment.getExecutionEnvironment();
- }
- return streamEnv;
- }
-
- /**
- * Get the BatchTableEnvironment. if the BatchTableEnvironment has not been set, it initial the
- * BatchTableEnvironment with default Configuration.
- *
- * @return the {@link BatchTableEnvironment}
- */
- public BatchTableEnvironment getBatchTableEnvironment() {
- if (null == batchTableEnv) {
- batchTableEnv = BatchTableEnvironment.create(getExecutionEnvironment());
- }
- return batchTableEnv;
- }
-
- /**
- * Get the StreamTableEnvironment. if the StreamTableEnvironment has not been set, it initial
- * the StreamTableEnvironment with default Configuration.
- *
- * @return the {@link StreamTableEnvironment}
- */
- public StreamTableEnvironment getStreamTableEnvironment() {
- if (null == streamTableEnv) {
- streamTableEnv =
- StreamTableEnvironment.create(
- getStreamExecutionEnvironment(),
- EnvironmentSettings.newInstance().useOldPlanner().build());
- }
- return streamTableEnv;
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java
deleted file mode 100644
index 8d2526c..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/MLEnvironmentFactory.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common;
-
-import org.apache.flink.util.Preconditions;
-
-import java.util.HashMap;
-
-/**
- * Factory to get the MLEnvironment using a MLEnvironmentId.
- *
- * <p>The following code snippet shows how to interact with MLEnvironmentFactory.
- *
- * <pre>{@code
- * long mlEnvId = MLEnvironmentFactory.getNewMLEnvironmentId();
- * MLEnvironment mlEnv = MLEnvironmentFactory.get(mlEnvId);
- * }</pre>
- */
-public class MLEnvironmentFactory {
-
- /** The default MLEnvironmentId. */
- public static final Long DEFAULT_ML_ENVIRONMENT_ID = 0L;
-
- /**
- * A monotonically increasing id for the MLEnvironments. Each id uniquely identifies an
- * MLEnvironment.
- */
- private static Long nextId = 1L;
-
- /** Map that hold the MLEnvironment and use the MLEnvironmentId as its key. */
- private static final HashMap<Long, MLEnvironment> map = new HashMap<>();
-
- static {
- map.put(DEFAULT_ML_ENVIRONMENT_ID, new MLEnvironment());
- }
-
- /**
- * Get the MLEnvironment using a MLEnvironmentId.
- *
- * @param mlEnvId the MLEnvironmentId
- * @return the MLEnvironment
- */
- public static synchronized MLEnvironment get(Long mlEnvId) {
- if (!map.containsKey(mlEnvId)) {
- throw new IllegalArgumentException(
- String.format(
- "Cannot find MLEnvironment for MLEnvironmentId %s."
- + " Did you get the MLEnvironmentId by calling getNewMLEnvironmentId?",
- mlEnvId));
- }
-
- return map.get(mlEnvId);
- }
-
- /**
- * Get the MLEnvironment use the default MLEnvironmentId.
- *
- * @return the default MLEnvironment.
- */
- public static synchronized MLEnvironment getDefault() {
- return get(DEFAULT_ML_ENVIRONMENT_ID);
- }
-
- /**
- * Create a unique MLEnvironment id and register a new MLEnvironment in the factory.
- *
- * @return the MLEnvironment id.
- */
- public static synchronized Long getNewMLEnvironmentId() {
- return registerMLEnvironment(new MLEnvironment());
- }
-
- /**
- * Register a new MLEnvironment to the factory and return a new MLEnvironment id.
- *
- * @param env the MLEnvironment that will be stored in the factory.
- * @return the MLEnvironment id.
- */
- public static synchronized Long registerMLEnvironment(MLEnvironment env) {
- map.put(nextId, env);
- return nextId++;
- }
-
- /**
- * Remove the MLEnvironment using the MLEnvironmentId.
- *
- * @param mlEnvId the id.
- * @return the removed MLEnvironment
- */
- public static synchronized MLEnvironment remove(Long mlEnvId) {
- Preconditions.checkNotNull(mlEnvId, "The environment id cannot be null.");
- // Never remove the default MLEnvironment. Just return the default environment.
- if (DEFAULT_ML_ENVIRONMENT_ID.equals(mlEnvId)) {
- return getDefault();
- } else {
- return map.remove(mlEnvId);
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/BLAS.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/BLAS.java
deleted file mode 100644
index aca2f23..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/BLAS.java
+++ /dev/null
@@ -1,234 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.apache.flink.util.Preconditions;
-
-/** A utility class that provides BLAS routines over matrices and vectors. */
-public class BLAS {
-
- /** For level-1 routines, we use Java implementation. */
- private static final com.github.fommil.netlib.BLAS NATIVE_BLAS =
- com.github.fommil.netlib.BLAS.getInstance();
-
- /**
- * For level-2 and level-3 routines, we use the native BLAS.
- *
- * <p>The NATIVE_BLAS instance tries to load BLAS implementations in the order: 1) optimized
- * system libraries such as Intel MKL, 2) self-contained native builds using the reference
- * Fortran from netlib.org, 3) F2J implementation. If to use optimized system libraries, it is
- * important to turn of their multi-thread support. Otherwise, it will conflict with Flink's
- * executor and leads to performance loss.
- */
- private static final com.github.fommil.netlib.BLAS F2J_BLAS =
- com.github.fommil.netlib.F2jBLAS.getInstance();
-
- /** \sum_i |x_i| . */
- public static double asum(int n, double[] x, int offset) {
- return F2J_BLAS.dasum(n, x, offset, 1);
- }
-
- /** \sum_i |x_i| . */
- public static double asum(DenseVector x) {
- return asum(x.data.length, x.data, 0);
- }
-
- /** \sum_i |x_i| . */
- public static double asum(SparseVector x) {
- return asum(x.values.length, x.values, 0);
- }
-
- /** y += a * x . */
- public static void axpy(double a, double[] x, double[] y) {
- Preconditions.checkArgument(x.length == y.length, "Array dimension mismatched.");
- F2J_BLAS.daxpy(x.length, a, x, 1, y, 1);
- }
-
- /** y += a * x . */
- public static void axpy(double a, DenseVector x, DenseVector y) {
- Preconditions.checkArgument(x.data.length == y.data.length, "Vector dimension mismatched.");
- F2J_BLAS.daxpy(x.data.length, a, x.data, 1, y.data, 1);
- }
-
- /** y += a * x . */
- public static void axpy(double a, SparseVector x, DenseVector y) {
- for (int i = 0; i < x.indices.length; i++) {
- y.data[x.indices[i]] += a * x.values[i];
- }
- }
-
- /** y += a * x . */
- public static void axpy(double a, DenseMatrix x, DenseMatrix y) {
- Preconditions.checkArgument(x.m == y.m && x.n == y.n, "Matrix dimension mismatched.");
- F2J_BLAS.daxpy(x.data.length, a, x.data, 1, y.data, 1);
- }
-
- /** y[yOffset:yOffset+n] += a * x[xOffset:xOffset+n] . */
- public static void axpy(int n, double a, double[] x, int xOffset, double[] y, int yOffset) {
- F2J_BLAS.daxpy(n, a, x, xOffset, 1, y, yOffset, 1);
- }
-
- /** x \cdot y . */
- public static double dot(double[] x, double[] y) {
- Preconditions.checkArgument(x.length == y.length, "Array dimension mismatched.");
- double s = 0.;
- for (int i = 0; i < x.length; i++) {
- s += x[i] * y[i];
- }
- return s;
- }
-
- /** x \cdot y . */
- public static double dot(DenseVector x, DenseVector y) {
- return dot(x.getData(), y.getData());
- }
-
- /** x = x * a . */
- public static void scal(double a, double[] x) {
- F2J_BLAS.dscal(x.length, a, x, 1);
- }
-
- /** x = x * a . */
- public static void scal(double a, DenseVector x) {
- F2J_BLAS.dscal(x.data.length, a, x.data, 1);
- }
-
- /** x = x * a . */
- public static void scal(double a, SparseVector x) {
- F2J_BLAS.dscal(x.values.length, a, x.values, 1);
- }
-
- /** x = x * a . */
- public static void scal(double a, DenseMatrix x) {
- F2J_BLAS.dscal(x.data.length, a, x.data, 1);
- }
-
- /** C := alpha * A * B + beta * C . */
- public static void gemm(
- double alpha,
- DenseMatrix matA,
- boolean transA,
- DenseMatrix matB,
- boolean transB,
- double beta,
- DenseMatrix matC) {
- if (transA) {
- Preconditions.checkArgument(
- matA.numCols() == matC.numRows(),
- "The columns of A does not match the rows of C");
- } else {
- Preconditions.checkArgument(
- matA.numRows() == matC.numRows(), "The rows of A does not match the rows of C");
- }
- if (transB) {
- Preconditions.checkArgument(
- matB.numRows() == matC.numCols(),
- "The rows of B does not match the columns of C");
- } else {
- Preconditions.checkArgument(
- matB.numCols() == matC.numCols(),
- "The columns of B does not match the columns of C");
- }
-
- final int m = matC.numRows();
- final int n = matC.numCols();
- final int k = transA ? matA.numRows() : matA.numCols();
- final int lda = matA.numRows();
- final int ldb = matB.numRows();
- final int ldc = matC.numRows();
- final String ta = transA ? "T" : "N";
- final String tb = transB ? "T" : "N";
- NATIVE_BLAS.dgemm(
- ta,
- tb,
- m,
- n,
- k,
- alpha,
- matA.getData(),
- lda,
- matB.getData(),
- ldb,
- beta,
- matC.getData(),
- ldc);
- }
-
- /** Check the compatibility of matrix and vector sizes in <code>gemv</code>. */
- private static void gemvDimensionCheck(DenseMatrix matA, boolean transA, Vector x, Vector y) {
- if (transA) {
- Preconditions.checkArgument(
- matA.numCols() == y.size() && matA.numRows() == x.size(),
- "Matrix and vector size mismatched.");
- } else {
- Preconditions.checkArgument(
- matA.numRows() == y.size() && matA.numCols() == x.size(),
- "Matrix and vector size mismatched.");
- }
- }
-
- /** y := alpha * A * x + beta * y . */
- public static void gemv(
- double alpha,
- DenseMatrix matA,
- boolean transA,
- DenseVector x,
- double beta,
- DenseVector y) {
- gemvDimensionCheck(matA, transA, x, y);
- final int m = matA.numRows();
- final int n = matA.numCols();
- final int lda = matA.numRows();
- final String ta = transA ? "T" : "N";
- NATIVE_BLAS.dgemv(
- ta, m, n, alpha, matA.getData(), lda, x.getData(), 1, beta, y.getData(), 1);
- }
-
- /** y := alpha * A * x + beta * y . */
- public static void gemv(
- double alpha,
- DenseMatrix matA,
- boolean transA,
- SparseVector x,
- double beta,
- DenseVector y) {
- gemvDimensionCheck(matA, transA, x, y);
- final int m = matA.numRows();
- final int n = matA.numCols();
- if (transA) {
- int start = 0;
- for (int i = 0; i < n; i++) {
- double s = 0.;
- for (int j = 0; j < x.indices.length; j++) {
- s += x.values[j] * matA.data[start + x.indices[j]];
- }
- y.data[i] = beta * y.data[i] + alpha * s;
- start += m;
- }
- } else {
- scal(beta, y);
- for (int i = 0; i < x.indices.length; i++) {
- int index = x.indices[i];
- double value = alpha * x.values[i];
- F2J_BLAS.daxpy(m, value, matA.data, index * m, 1, y.data, 0, 1);
- }
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseMatrix.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseMatrix.java
deleted file mode 100644
index 643416c..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseMatrix.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import java.io.Serializable;
-import java.util.Arrays;
-
-/**
- * DenseMatrix stores dense matrix data and provides some methods to operate on the matrix it
- * represents.
- */
-public class DenseMatrix implements Serializable {
-
- /**
- * Row dimension.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- */
- int m;
-
- /**
- * Column dimension.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- */
- int n;
-
- /**
- * Array for internal storage of elements.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- *
- * <p>The matrix data is stored in column major format internally.
- */
- double[] data;
-
- /**
- * Construct an m-by-n matrix of zeros.
- *
- * @param m Number of rows.
- * @param n Number of columns.
- */
- public DenseMatrix(int m, int n) {
- this(m, n, new double[m * n], false);
- }
-
- /**
- * Construct a matrix from a 1-D array. The data in the array should organize in column major.
- *
- * @param m Number of rows.
- * @param n Number of cols.
- * @param data One-dimensional array of doubles.
- */
- public DenseMatrix(int m, int n, double[] data) {
- this(m, n, data, false);
- }
-
- /**
- * Construct a matrix from a 1-D array. The data in the array is organized in column major or in
- * row major, which is specified by parameter 'inRowMajor'
- *
- * @param m Number of rows.
- * @param n Number of cols.
- * @param data One-dimensional array of doubles.
- * @param inRowMajor Whether the matrix in 'data' is in row major format.
- */
- public DenseMatrix(int m, int n, double[] data, boolean inRowMajor) {
- assert (data.length == m * n);
- this.m = m;
- this.n = n;
- if (inRowMajor) {
- toColumnMajor(m, n, data);
- }
- this.data = data;
- }
-
- /**
- * Construct a matrix from a 2-D array.
- *
- * @param data Two-dimensional array of doubles.
- * @throws IllegalArgumentException All rows must have the same size
- */
- public DenseMatrix(double[][] data) {
- this.m = data.length;
- if (this.m == 0) {
- this.n = 0;
- this.data = new double[0];
- return;
- }
- this.n = data[0].length;
- for (int i = 0; i < m; i++) {
- if (data[i].length != n) {
- throw new IllegalArgumentException("All rows must have the same size.");
- }
- }
- this.data = new double[m * n];
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- this.set(i, j, data[i][j]);
- }
- }
- }
-
- /**
- * Create an identity matrix.
- *
- * @param n the dimension of the eye matrix.
- * @return an identity matrix.
- */
- public static DenseMatrix eye(int n) {
- return eye(n, n);
- }
-
- /**
- * Create a m * n identity matrix.
- *
- * @param m the row dimension.
- * @param n the column dimension.e
- * @return the m * n identity matrix.
- */
- public static DenseMatrix eye(int m, int n) {
- DenseMatrix mat = new DenseMatrix(m, n);
- int k = Math.min(m, n);
- for (int i = 0; i < k; i++) {
- mat.data[i * m + i] = 1.0;
- }
- return mat;
- }
-
- /**
- * Create a zero matrix.
- *
- * @param m the row dimension.
- * @param n the column dimension.
- * @return a m * n zero matrix.
- */
- public static DenseMatrix zeros(int m, int n) {
- return new DenseMatrix(m, n);
- }
-
- /**
- * Create a matrix with all elements set to 1.
- *
- * @param m the row dimension
- * @param n the column dimension
- * @return the m * n matrix with all elements set to 1.
- */
- public static DenseMatrix ones(int m, int n) {
- DenseMatrix mat = new DenseMatrix(m, n);
- Arrays.fill(mat.data, 1.);
- return mat;
- }
-
- /**
- * Create a random matrix.
- *
- * @param m the row dimension
- * @param n the column dimension.
- * @return a m * n random matrix.
- */
- public static DenseMatrix rand(int m, int n) {
- DenseMatrix mat = new DenseMatrix(m, n);
- for (int i = 0; i < mat.data.length; i++) {
- mat.data[i] = Math.random();
- }
- return mat;
- }
-
- /**
- * Create a random symmetric matrix.
- *
- * @param n the dimension of the symmetric matrix.
- * @return a n * n random symmetric matrix.
- */
- public static DenseMatrix randSymmetric(int n) {
- DenseMatrix mat = new DenseMatrix(n, n);
- for (int i = 0; i < n; i++) {
- for (int j = i; j < n; j++) {
- double r = Math.random();
- mat.set(i, j, r);
- if (i != j) {
- mat.set(j, i, r);
- }
- }
- }
- return mat;
- }
-
- /**
- * Get a single element.
- *
- * @param i Row index.
- * @param j Column index.
- * @return matA(i, j)
- * @throws ArrayIndexOutOfBoundsException
- */
- public double get(int i, int j) {
- return data[j * m + i];
- }
-
- /**
- * Get the data array of this matrix.
- *
- * @return the data array of this matrix.
- */
- public double[] getData() {
- return this.data;
- }
-
- /**
- * Get all the matrix data, returned as a 2-D array.
- *
- * @return all matrix data, returned as a 2-D array.
- */
- public double[][] getArrayCopy2D() {
- double[][] arrayData = new double[m][n];
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- arrayData[i][j] = this.get(i, j);
- }
- }
- return arrayData;
- }
-
- /**
- * Get all matrix data, returned as a 1-D array.
- *
- * @param inRowMajor Whether to return data in row major.
- * @return all matrix data, returned as a 1-D array.
- */
- public double[] getArrayCopy1D(boolean inRowMajor) {
- if (inRowMajor) {
- double[] arrayData = new double[m * n];
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- arrayData[i * n + j] = this.get(i, j);
- }
- }
- return arrayData;
- } else {
- return this.data.clone();
- }
- }
-
- /**
- * Get one row.
- *
- * @param row the row index.
- * @return the row with the given index.
- */
- public double[] getRow(int row) {
- assert (row >= 0 && row < m) : "Invalid row index.";
- double[] r = new double[n];
- for (int i = 0; i < n; i++) {
- r[i] = this.get(row, i);
- }
- return r;
- }
-
- /**
- * Get one column.
- *
- * @param col the column index.
- * @return the column with the given index.
- */
- public double[] getColumn(int col) {
- assert (col >= 0 && col < n) : "Invalid column index.";
- double[] columnData = new double[m];
- System.arraycopy(this.data, col * m, columnData, 0, m);
- return columnData;
- }
-
- /** Clone the Matrix object. */
- @Override
- public DenseMatrix clone() {
- return new DenseMatrix(this.m, this.n, this.data.clone(), false);
- }
-
- /**
- * Create a new matrix by selecting some of the rows.
- *
- * @param rows the array of row indexes to select.
- * @return a new matrix by selecting some of the rows.
- */
- public DenseMatrix selectRows(int[] rows) {
- DenseMatrix sub = new DenseMatrix(rows.length, this.n);
- for (int i = 0; i < rows.length; i++) {
- for (int j = 0; j < this.n; j++) {
- sub.set(i, j, this.get(rows[i], j));
- }
- }
- return sub;
- }
-
- /**
- * Get sub matrix.
- *
- * @param m0 the starting row index (inclusive)
- * @param m1 the ending row index (exclusive)
- * @param n0 the starting column index (inclusive)
- * @param n1 the ending column index (exclusive)
- * @return the specified sub matrix.
- */
- public DenseMatrix getSubMatrix(int m0, int m1, int n0, int n1) {
- assert (m0 >= 0 && m1 <= m) && (n0 >= 0 && n1 <= n) : "Invalid index range.";
- DenseMatrix sub = new DenseMatrix(m1 - m0, n1 - n0);
- for (int i = 0; i < sub.m; i++) {
- for (int j = 0; j < sub.n; j++) {
- sub.set(i, j, this.get(m0 + i, n0 + j));
- }
- }
- return sub;
- }
-
- /**
- * Set part of the matrix values from the values of another matrix.
- *
- * @param sub the matrix whose element values will be assigned to the sub matrix of this matrix.
- * @param m0 the starting row index (inclusive)
- * @param m1 the ending row index (exclusive)
- * @param n0 the starting column index (inclusive)
- * @param n1 the ending column index (exclusive)
- */
- public void setSubMatrix(DenseMatrix sub, int m0, int m1, int n0, int n1) {
- assert (m0 >= 0 && m1 <= m) && (n0 >= 0 && n1 <= n) : "Invalid index range.";
- for (int i = 0; i < sub.m; i++) {
- for (int j = 0; j < sub.n; j++) {
- this.set(m0 + i, n0 + j, sub.get(i, j));
- }
- }
- }
-
- /**
- * Set a single element.
- *
- * @param i Row index.
- * @param j Column index.
- * @param s A(i,j).
- * @throws ArrayIndexOutOfBoundsException
- */
- public void set(int i, int j, double s) {
- data[j * m + i] = s;
- }
-
- /**
- * Add the given value to a single element.
- *
- * @param i Row index.
- * @param j Column index.
- * @param s A(i,j).
- * @throws ArrayIndexOutOfBoundsException
- */
- public void add(int i, int j, double s) {
- data[j * m + i] += s;
- }
-
- /**
- * Check whether the matrix is square matrix.
- *
- * @return <code>true</code> if this matrix is a square matrix, <code>false</code> otherwise.
- */
- public boolean isSquare() {
- return m == n;
- }
-
- /**
- * Check whether the matrix is symmetric matrix.
- *
- * @return <code>true</code> if this matrix is a symmetric matrix, <code>false</code> otherwise.
- */
- public boolean isSymmetric() {
- if (m != n) {
- return false;
- }
- for (int i = 0; i < n; i++) {
- for (int j = i + 1; j < n; j++) {
- if (this.get(i, j) != this.get(j, i)) {
- return false;
- }
- }
- }
- return true;
- }
-
- /**
- * Get the number of rows.
- *
- * @return the number of rows.
- */
- public int numRows() {
- return m;
- }
-
- /**
- * Get the number of columns.
- *
- * @return the number of columns.
- */
- public int numCols() {
- return n;
- }
-
- /** Sum of all elements of the matrix. */
- public double sum() {
- double s = 0.;
- for (int i = 0; i < this.data.length; i++) {
- s += this.data[i];
- }
- return s;
- }
-
- /** Scale the vector by value "v" and create a new matrix to store the result. */
- public DenseMatrix scale(double v) {
- DenseMatrix r = this.clone();
- BLAS.scal(v, r);
- return r;
- }
-
- /** Scale the matrix by value "v". */
- public void scaleEqual(double v) {
- BLAS.scal(v, this);
- }
-
- /** Create a new matrix by plussing another matrix. */
- public DenseMatrix plus(DenseMatrix mat) {
- DenseMatrix r = this.clone();
- BLAS.axpy(1.0, mat, r);
- return r;
- }
-
- /** Create a new matrix by plussing a constant. */
- public DenseMatrix plus(double alpha) {
- DenseMatrix r = this.clone();
- for (int i = 0; i < r.data.length; i++) {
- r.data[i] += alpha;
- }
- return r;
- }
-
- /** Plus with another matrix. */
- public void plusEquals(DenseMatrix mat) {
- BLAS.axpy(1.0, mat, this);
- }
-
- /** Plus with a constant. */
- public void plusEquals(double alpha) {
- for (int i = 0; i < this.data.length; i++) {
- this.data[i] += alpha;
- }
- }
-
- /** Create a new matrix by subtracting another matrix. */
- public DenseMatrix minus(DenseMatrix mat) {
- DenseMatrix r = this.clone();
- BLAS.axpy(-1.0, mat, r);
- return r;
- }
-
- /** Minus with another vector. */
- public void minusEquals(DenseMatrix mat) {
- BLAS.axpy(-1.0, mat, this);
- }
-
- /** Multiply with another matrix. */
- public DenseMatrix multiplies(DenseMatrix mat) {
- DenseMatrix r = new DenseMatrix(this.m, mat.n);
- BLAS.gemm(1.0, this, false, mat, false, 0., r);
- return r;
- }
-
- /** Multiply with a dense vector. */
- public DenseVector multiplies(DenseVector x) {
- DenseVector y = new DenseVector(this.numRows());
- BLAS.gemv(1.0, this, false, x, 0.0, y);
- return y;
- }
-
- /** Multiply with a sparse vector. */
- public DenseVector multiplies(SparseVector x) {
- DenseVector y = new DenseVector(this.numRows());
- for (int i = 0; i < this.numRows(); i++) {
- double s = 0.;
- int[] indices = x.getIndices();
- double[] values = x.getValues();
- for (int j = 0; j < indices.length; j++) {
- int index = indices[j];
- if (index >= this.numCols()) {
- throw new RuntimeException("Vector index out of bound:" + index);
- }
- s += this.get(i, index) * values[j];
- }
- y.set(i, s);
- }
- return y;
- }
-
- /**
- * Create a new matrix by transposing current matrix.
- *
- * <p>Use cache-oblivious matrix transpose algorithm.
- */
- public DenseMatrix transpose() {
- DenseMatrix mat = new DenseMatrix(n, m);
- int m0 = m;
- int n0 = n;
- int barrierSize = 16384;
- while (m0 * n0 > barrierSize) {
- if (m0 >= n0) {
- m0 /= 2;
- } else {
- n0 /= 2;
- }
- }
- for (int i0 = 0; i0 < m; i0 += m0) {
- for (int j0 = 0; j0 < n; j0 += n0) {
- for (int i = i0; i < i0 + m0 && i < m; i++) {
- for (int j = j0; j < j0 + n0 && j < n; j++) {
- mat.set(j, i, this.get(i, j));
- }
- }
- }
- }
- return mat;
- }
-
- /** Converts the data layout in "data" from row major to column major. */
- private static void toColumnMajor(int m, int n, double[] data) {
- if (m == n) {
- for (int i = 0; i < m; i++) {
- for (int j = i + 1; j < m; j++) {
- int pos0 = j * m + i;
- int pos1 = i * m + j;
- double t = data[pos0];
- data[pos0] = data[pos1];
- data[pos1] = t;
- }
- }
- } else {
- DenseMatrix temp = new DenseMatrix(n, m, data, false);
- System.arraycopy(temp.transpose().data, 0, data, 0, data.length);
- }
- }
-
- @Override
- public String toString() {
- StringBuilder sbd = new StringBuilder();
- sbd.append(String.format("mat[%d,%d]:\n", m, n));
- for (int i = 0; i < m; i++) {
- sbd.append(" ");
- for (int j = 0; j < n; j++) {
- if (j > 0) {
- sbd.append(",");
- }
- sbd.append(this.get(i, j));
- }
- sbd.append("\n");
- }
- return sbd.toString();
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseVector.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseVector.java
deleted file mode 100644
index 951cb49..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/DenseVector.java
+++ /dev/null
@@ -1,379 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import java.util.Arrays;
-import java.util.Random;
-
-/** A dense vector represented by a values array. */
-public class DenseVector extends Vector {
- /**
- * The array holding the vector data.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- */
- double[] data;
-
- /** Create a zero size vector. */
- public DenseVector() {
- this(0);
- }
-
- /**
- * Create a size n vector with all elements zero.
- *
- * @param n Size of the vector.
- */
- public DenseVector(int n) {
- this.data = new double[n];
- }
-
- /**
- * Create a dense vector with the user provided data.
- *
- * @param data The vector data.
- */
- public DenseVector(double[] data) {
- this.data = data;
- }
-
- /** Get the data array. */
- public double[] getData() {
- return this.data;
- }
-
- /** Set the data array. */
- public void setData(double[] data) {
- this.data = data;
- }
-
- /**
- * Create a dense vector with all elements one.
- *
- * @param n Size of the vector.
- * @return The newly created dense vector.
- */
- public static DenseVector ones(int n) {
- DenseVector r = new DenseVector(n);
- Arrays.fill(r.data, 1.0);
- return r;
- }
-
- /**
- * Create a dense vector with all elements zero.
- *
- * @param n Size of the vector.
- * @return The newly created dense vector.
- */
- public static DenseVector zeros(int n) {
- DenseVector r = new DenseVector(n);
- Arrays.fill(r.data, 0.0);
- return r;
- }
-
- /**
- * Create a dense vector with random values uniformly distributed in the range of [0.0, 1.0].
- *
- * @param n Size of the vector.
- * @return The newly created dense vector.
- */
- public static DenseVector rand(int n) {
- Random random = new Random();
- DenseVector v = new DenseVector(n);
- for (int i = 0; i < n; i++) {
- v.data[i] = random.nextDouble();
- }
- return v;
- }
-
- @Override
- public DenseVector clone() {
- return new DenseVector(this.data.clone());
- }
-
- @Override
- public String toString() {
- return VectorUtil.toString(this);
- }
-
- @Override
- public int size() {
- return data.length;
- }
-
- @Override
- public double get(int i) {
- return data[i];
- }
-
- @Override
- public void set(int i, double d) {
- data[i] = d;
- }
-
- @Override
- public void add(int i, double d) {
- data[i] += d;
- }
-
- @Override
- public double normL1() {
- double d = 0;
- for (double t : data) {
- d += Math.abs(t);
- }
- return d;
- }
-
- @Override
- public double normL2() {
- double d = 0;
- for (double t : data) {
- d += t * t;
- }
- return Math.sqrt(d);
- }
-
- @Override
- public double normL2Square() {
- double d = 0;
- for (double t : data) {
- d += t * t;
- }
- return d;
- }
-
- @Override
- public double normInf() {
- double d = 0;
- for (double t : data) {
- d = Math.max(Math.abs(t), d);
- }
- return d;
- }
-
- @Override
- public DenseVector slice(int[] indices) {
- double[] values = new double[indices.length];
- for (int i = 0; i < indices.length; ++i) {
- if (indices[i] >= data.length) {
- throw new RuntimeException("Index is larger than vector size.");
- }
- values[i] = data[indices[i]];
- }
- return new DenseVector(values);
- }
-
- @Override
- public DenseVector prefix(double d) {
- double[] data = new double[this.size() + 1];
- data[0] = d;
- System.arraycopy(this.data, 0, data, 1, this.data.length);
- return new DenseVector(data);
- }
-
- @Override
- public DenseVector append(double d) {
- double[] data = new double[this.size() + 1];
- System.arraycopy(this.data, 0, data, 0, this.data.length);
- data[this.size()] = d;
- return new DenseVector(data);
- }
-
- @Override
- public void scaleEqual(double d) {
- BLAS.scal(d, this);
- }
-
- @Override
- public DenseVector plus(Vector other) {
- DenseVector r = this.clone();
- if (other instanceof DenseVector) {
- BLAS.axpy(1.0, (DenseVector) other, r);
- } else {
- BLAS.axpy(1.0, (SparseVector) other, r);
- }
- return r;
- }
-
- @Override
- public DenseVector minus(Vector other) {
- DenseVector r = this.clone();
- if (other instanceof DenseVector) {
- BLAS.axpy(-1.0, (DenseVector) other, r);
- } else {
- BLAS.axpy(-1.0, (SparseVector) other, r);
- }
- return r;
- }
-
- @Override
- public DenseVector scale(double d) {
- DenseVector r = this.clone();
- BLAS.scal(d, r);
- return r;
- }
-
- @Override
- public double dot(Vector vec) {
- if (vec instanceof DenseVector) {
- return BLAS.dot(this, (DenseVector) vec);
- } else {
- return vec.dot(this);
- }
- }
-
- @Override
- public void standardizeEqual(double mean, double stdvar) {
- int size = data.length;
- for (int i = 0; i < size; i++) {
- data[i] -= mean;
- data[i] *= (1.0 / stdvar);
- }
- }
-
- @Override
- public void normalizeEqual(double p) {
- double norm = 0.0;
- if (Double.isInfinite(p)) {
- norm = normInf();
- } else if (p == 1.0) {
- norm = normL1();
- } else if (p == 2.0) {
- norm = normL2();
- } else {
- for (int i = 0; i < data.length; i++) {
- norm += Math.pow(Math.abs(data[i]), p);
- }
- norm = Math.pow(norm, 1 / p);
- }
- for (int i = 0; i < data.length; i++) {
- data[i] /= norm;
- }
- }
-
- /** Set the data of the vector the same as those of another vector. */
- public void setEqual(DenseVector other) {
- assert this.size() == other.size() : "Size of the two vectors mismatched.";
- System.arraycopy(other.data, 0, this.data, 0, this.size());
- }
-
- /** Plus with another vector. */
- public void plusEqual(Vector other) {
- if (other instanceof DenseVector) {
- BLAS.axpy(1.0, (DenseVector) other, this);
- } else {
- BLAS.axpy(1.0, (SparseVector) other, this);
- }
- }
-
- /** Minus with another vector. */
- public void minusEqual(Vector other) {
- if (other instanceof DenseVector) {
- BLAS.axpy(-1.0, (DenseVector) other, this);
- } else {
- BLAS.axpy(-1.0, (SparseVector) other, this);
- }
- }
-
- /** Plus with another vector scaled by "alpha". */
- public void plusScaleEqual(Vector other, double alpha) {
- if (other instanceof DenseVector) {
- BLAS.axpy(alpha, (DenseVector) other, this);
- } else {
- BLAS.axpy(alpha, (SparseVector) other, this);
- }
- }
-
- @Override
- public DenseMatrix outer() {
- return this.outer(this);
- }
-
- /**
- * Compute the outer product with another vector.
- *
- * @return The outer product matrix.
- */
- public DenseMatrix outer(DenseVector other) {
- int nrows = this.size();
- int ncols = other.size();
- double[] data = new double[nrows * ncols];
- int pos = 0;
- for (int j = 0; j < ncols; j++) {
- for (int i = 0; i < nrows; i++) {
- data[pos++] = this.data[i] * other.data[j];
- }
- }
- return new DenseMatrix(nrows, ncols, data, false);
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- DenseVector that = (DenseVector) o;
- return Arrays.equals(data, that.data);
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(data);
- }
-
- @Override
- public VectorIterator iterator() {
- return new DenseVectorIterator();
- }
-
- private class DenseVectorIterator implements VectorIterator {
- private int cursor = 0;
-
- @Override
- public boolean hasNext() {
- return cursor < data.length;
- }
-
- @Override
- public void next() {
- cursor++;
- }
-
- @Override
- public int getIndex() {
- if (cursor >= data.length) {
- throw new RuntimeException("Iterator out of bound.");
- }
- return cursor;
- }
-
- @Override
- public double getValue() {
- if (cursor >= data.length) {
- throw new RuntimeException("Iterator out of bound.");
- }
- return data[cursor];
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/MatVecOp.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/MatVecOp.java
deleted file mode 100644
index fafb6f2..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/MatVecOp.java
+++ /dev/null
@@ -1,307 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import java.util.function.BiFunction;
-import java.util.function.Function;
-
-/**
- * A utility class that provides operations over {@link DenseVector}, {@link SparseVector} and
- * {@link DenseMatrix}.
- */
-public class MatVecOp {
- /** compute vec1 + vec2 . */
- public static Vector plus(Vector vec1, Vector vec2) {
- return vec1.plus(vec2);
- }
-
- /** compute vec1 - vec2 . */
- public static Vector minus(Vector vec1, Vector vec2) {
- return vec1.minus(vec2);
- }
-
- /** Compute vec1 \cdot vec2 . */
- public static double dot(Vector vec1, Vector vec2) {
- return vec1.dot(vec2);
- }
-
- /** Compute || vec1 - vec2 ||_1 . */
- public static double sumAbsDiff(Vector vec1, Vector vec2) {
- if (vec1 instanceof DenseVector) {
- if (vec2 instanceof DenseVector) {
- return MatVecOp.applySum(
- (DenseVector) vec1, (DenseVector) vec2, (a, b) -> Math.abs(a - b));
- } else {
- return MatVecOp.applySum(
- (DenseVector) vec1, (SparseVector) vec2, (a, b) -> Math.abs(a - b));
- }
- } else {
- if (vec2 instanceof DenseVector) {
- return MatVecOp.applySum(
- (SparseVector) vec1, (DenseVector) vec2, (a, b) -> Math.abs(a - b));
- } else {
- return MatVecOp.applySum(
- (SparseVector) vec1, (SparseVector) vec2, (a, b) -> Math.abs(a - b));
- }
- }
- }
-
- /** Compute || vec1 - vec2 ||_2^2 . */
- public static double sumSquaredDiff(Vector vec1, Vector vec2) {
- if (vec1 instanceof DenseVector) {
- if (vec2 instanceof DenseVector) {
- return MatVecOp.applySum(
- (DenseVector) vec1, (DenseVector) vec2, (a, b) -> (a - b) * (a - b));
- } else {
- return MatVecOp.applySum(
- (DenseVector) vec1, (SparseVector) vec2, (a, b) -> (a - b) * (a - b));
- }
- } else {
- if (vec2 instanceof DenseVector) {
- return MatVecOp.applySum(
- (SparseVector) vec1, (DenseVector) vec2, (a, b) -> (a - b) * (a - b));
- } else {
- return MatVecOp.applySum(
- (SparseVector) vec1, (SparseVector) vec2, (a, b) -> (a - b) * (a - b));
- }
- }
- }
-
- /** y = func(x). */
- public static void apply(DenseMatrix x, DenseMatrix y, Function<Double, Double> func) {
- assert (x.m == y.m && x.n == y.n) : "x and y size mismatched.";
- double[] xdata = x.data;
- double[] ydata = y.data;
- for (int i = 0; i < xdata.length; i++) {
- ydata[i] = func.apply(xdata[i]);
- }
- }
-
- /** y = func(x1, x2). */
- public static void apply(
- DenseMatrix x1,
- DenseMatrix x2,
- DenseMatrix y,
- BiFunction<Double, Double, Double> func) {
-
- assert (x1.m == y.m && x1.n == y.n) : "x1 and y size mismatched.";
- assert (x2.m == y.m && x2.n == y.n) : "x2 and y size mismatched.";
- double[] x1data = x1.data;
- double[] x2data = x2.data;
- double[] ydata = y.data;
- for (int i = 0; i < ydata.length; i++) {
- ydata[i] = func.apply(x1data[i], x2data[i]);
- }
- }
-
- /** y = func(x). */
- public static void apply(DenseVector x, DenseVector y, Function<Double, Double> func) {
- assert (x.data.length == y.data.length) : "x and y size mismatched.";
- for (int i = 0; i < x.data.length; i++) {
- y.data[i] = func.apply(x.data[i]);
- }
- }
-
- /** y = func(x1, x2). */
- public static void apply(
- DenseVector x1,
- DenseVector x2,
- DenseVector y,
- BiFunction<Double, Double, Double> func) {
-
- assert (x1.data.length == y.data.length) : "x1 and y size mismatched.";
- assert (x2.data.length == y.data.length) : "x1 and y size mismatched.";
- for (int i = 0; i < y.data.length; i++) {
- y.data[i] = func.apply(x1.data[i], x2.data[i]);
- }
- }
-
- /**
- * Create a new {@link SparseVector} by element wise operation between two {@link
- * SparseVector}s. y = func(x1, x2).
- */
- public static SparseVector apply(
- SparseVector x1, SparseVector x2, BiFunction<Double, Double, Double> func) {
- assert (x1.size() == x2.size()) : "x1 and x2 size mismatched.";
-
- int totNnz = x1.values.length + x2.values.length;
- int p0 = 0;
- int p1 = 0;
- while (p0 < x1.values.length && p1 < x2.values.length) {
- if (x1.indices[p0] == x2.indices[p1]) {
- totNnz--;
- p0++;
- p1++;
- } else if (x1.indices[p0] < x2.indices[p1]) {
- p0++;
- } else {
- p1++;
- }
- }
-
- SparseVector r = new SparseVector(x1.size());
- r.indices = new int[totNnz];
- r.values = new double[totNnz];
- p0 = p1 = 0;
- int pos = 0;
- while (pos < totNnz) {
- if (p0 < x1.values.length && p1 < x2.values.length) {
- if (x1.indices[p0] == x2.indices[p1]) {
- r.indices[pos] = x1.indices[p0];
- r.values[pos] = func.apply(x1.values[p0], x2.values[p1]);
- p0++;
- p1++;
- } else if (x1.indices[p0] < x2.indices[p1]) {
- r.indices[pos] = x1.indices[p0];
- r.values[pos] = func.apply(x1.values[p0], 0.0);
- p0++;
- } else {
- r.indices[pos] = x2.indices[p1];
- r.values[pos] = func.apply(0.0, x2.values[p1]);
- p1++;
- }
- pos++;
- } else {
- if (p0 < x1.values.length) {
- r.indices[pos] = x1.indices[p0];
- r.values[pos] = func.apply(x1.values[p0], 0.0);
- p0++;
- pos++;
- continue;
- }
- if (p1 < x2.values.length) {
- r.indices[pos] = x2.indices[p1];
- r.values[pos] = func.apply(0.0, x2.values[p1]);
- p1++;
- pos++;
- continue;
- }
- }
- }
-
- return r;
- }
-
- /** \sum_i func(x1_i, x2_i) . */
- public static double applySum(
- DenseVector x1, DenseVector x2, BiFunction<Double, Double, Double> func) {
- assert x1.size() == x2.size() : "x1 and x2 size mismatched.";
- double[] x1data = x1.getData();
- double[] x2data = x2.getData();
- double s = 0.;
- for (int i = 0; i < x1data.length; i++) {
- s += func.apply(x1data[i], x2data[i]);
- }
- return s;
- }
-
- /** \sum_i func(x1_i, x2_i) . */
- public static double applySum(
- SparseVector x1, SparseVector x2, BiFunction<Double, Double, Double> func) {
- double s = 0.;
- int p1 = 0;
- int p2 = 0;
- int[] x1Indices = x1.getIndices();
- double[] x1Values = x1.getValues();
- int[] x2Indices = x2.getIndices();
- double[] x2Values = x2.getValues();
- int nnz1 = x1Indices.length;
- int nnz2 = x2Indices.length;
- while (p1 < nnz1 || p2 < nnz2) {
- if (p1 < nnz1 && p2 < nnz2) {
- if (x1Indices[p1] == x2Indices[p2]) {
- s += func.apply(x1Values[p1], x2Values[p2]);
- p1++;
- p2++;
- } else if (x1Indices[p1] < x2Indices[p2]) {
- s += func.apply(x1Values[p1], 0.);
- p1++;
- } else {
- s += func.apply(0., x2Values[p2]);
- p2++;
- }
- } else {
- if (p1 < nnz1) {
- s += func.apply(x1Values[p1], 0.);
- p1++;
- } else { // p2 < nnz2
- s += func.apply(0., x2Values[p2]);
- p2++;
- }
- }
- }
- return s;
- }
-
- /** \sum_i func(x1_i, x2_i) . */
- public static double applySum(
- DenseVector x1, SparseVector x2, BiFunction<Double, Double, Double> func) {
- assert x1.size() == x2.size() : "x1 and x2 size mismatched.";
- double s = 0.;
- int p2 = 0;
- int[] x2Indices = x2.getIndices();
- double[] x2Values = x2.getValues();
- int nnz2 = x2Indices.length;
- double[] x1data = x1.getData();
- for (int i = 0; i < x1data.length; i++) {
- if (p2 < nnz2 && x2Indices[p2] == i) {
- s += func.apply(x1data[i], x2Values[p2]);
- p2++;
- } else {
- s += func.apply(x1data[i], 0.);
- }
- }
- return s;
- }
-
- /** \sum_i func(x1_i, x2_i) . */
- public static double applySum(
- SparseVector x1, DenseVector x2, BiFunction<Double, Double, Double> func) {
- assert x1.size() == x2.size() : "x1 and x2 size mismatched.";
- double s = 0.;
- int p1 = 0;
- int[] x1Indices = x1.getIndices();
- double[] x1Values = x1.getValues();
- int nnz1 = x1Indices.length;
- double[] x2data = x2.getData();
- for (int i = 0; i < x2data.length; i++) {
- if (p1 < nnz1 && x1Indices[p1] == i) {
- s += func.apply(x1Values[p1], x2data[i]);
- p1++;
- } else {
- s += func.apply(0., x2data[i]);
- }
- }
- return s;
- }
-
- /** \sum_ij func(x1_ij, x2_ij) . */
- public static double applySum(
- DenseMatrix x1, DenseMatrix x2, BiFunction<Double, Double, Double> func) {
- assert (x1.m == x2.m && x1.n == x2.n) : "x1 and x2 size mismatched.";
- double[] x1data = x1.data;
- double[] x2data = x2.data;
- double s = 0.;
- for (int i = 0; i < x1data.length; i++) {
- s += func.apply(x1data[i], x2data[i]);
- }
- return s;
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/SparseVector.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/SparseVector.java
deleted file mode 100644
index f11a188..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/SparseVector.java
+++ /dev/null
@@ -1,574 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.TreeMap;
-
-/** A sparse vector represented by an indices array and a values array. */
-public class SparseVector extends Vector {
-
- /**
- * Size of the vector. n = -1 indicates that the vector size is undetermined.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- */
- int n;
-
- /**
- * Column indices.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- */
- int[] indices;
-
- /**
- * Column values.
- *
- * <p>Package private to allow access from {@link MatVecOp} and {@link BLAS}.
- */
- double[] values;
-
- /** Construct an empty sparse vector with undetermined size. */
- public SparseVector() {
- this(-1);
- }
-
- /** Construct an empty sparse vector with determined size. */
- public SparseVector(int n) {
- this.n = n;
- this.indices = new int[0];
- this.values = new double[0];
- }
-
- /**
- * Construct a sparse vector with the given indices and values.
- *
- * @throws IllegalArgumentException If size of indices array and values array differ.
- * @throws IllegalArgumentException If n >= 0 and the indices are out of bound.
- */
- public SparseVector(int n, int[] indices, double[] values) {
- this.n = n;
- this.indices = indices;
- this.values = values;
- checkSizeAndIndicesRange();
- sortIndices();
- }
-
- /**
- * Construct a sparse vector with given indices to values map.
- *
- * @throws IllegalArgumentException If n >= 0 and the indices are out of bound.
- */
- public SparseVector(int n, Map<Integer, Double> kv) {
- this.n = n;
- int nnz = kv.size();
- int[] indices = new int[nnz];
- double[] values = new double[nnz];
-
- int pos = 0;
- for (Map.Entry<Integer, Double> entry : kv.entrySet()) {
- indices[pos] = entry.getKey();
- values[pos] = entry.getValue();
- pos++;
- }
-
- this.indices = indices;
- this.values = values;
- checkSizeAndIndicesRange();
-
- if (!(kv instanceof TreeMap)) {
- sortIndices();
- }
- }
-
- /**
- * Check whether the indices array and values array are of the same size, and whether vector
- * indices are in valid range.
- */
- private void checkSizeAndIndicesRange() {
- if (indices.length != values.length) {
- throw new IllegalArgumentException("Indices size and values size should be the same.");
- }
- for (int i = 0; i < indices.length; i++) {
- if (indices[i] < 0 || (n >= 0 && indices[i] >= n)) {
- throw new IllegalArgumentException("Index out of bound.");
- }
- }
- }
-
- /** Sort the indices and values using quick sort. */
- private static void sortImpl(int[] indices, double[] values, int low, int high) {
- int pivot = indices[high];
- int pos = low - 1;
- for (int i = low; i <= high; i++) {
- if (indices[i] <= pivot) {
- pos++;
- int tempI = indices[pos];
- double tempD = values[pos];
- indices[pos] = indices[i];
- values[pos] = values[i];
- indices[i] = tempI;
- values[i] = tempD;
- }
- }
- if (pos - 1 > low) {
- sortImpl(indices, values, low, pos - 1);
- }
- if (high > pos + 1) {
- sortImpl(indices, values, pos + 1, high);
- }
- }
-
- /** Sort the indices and values if the indices are out of order. */
- private void sortIndices() {
- boolean outOfOrder = false;
- for (int i = 0; i < this.indices.length - 1; i++) {
- if (this.indices[i] > this.indices[i + 1]) {
- outOfOrder = true;
- break;
- }
- }
- if (outOfOrder) {
- sortImpl(this.indices, this.values, 0, this.indices.length - 1);
- }
- }
-
- @Override
- public SparseVector clone() {
- SparseVector vec = new SparseVector(this.n);
- vec.indices = this.indices.clone();
- vec.values = this.values.clone();
- return vec;
- }
-
- @Override
- public SparseVector prefix(double d) {
- int[] indices = new int[this.indices.length + 1];
- double[] values = new double[this.values.length + 1];
- int n = (this.n >= 0) ? this.n + 1 : this.n;
-
- indices[0] = 0;
- values[0] = d;
-
- for (int i = 0; i < this.indices.length; i++) {
- indices[i + 1] = this.indices[i] + 1;
- values[i + 1] = this.values[i];
- }
-
- return new SparseVector(n, indices, values);
- }
-
- @Override
- public SparseVector append(double d) {
- int[] indices = new int[this.indices.length + 1];
- double[] values = new double[this.values.length + 1];
- int n = (this.n >= 0) ? this.n + 1 : this.n;
-
- System.arraycopy(this.indices, 0, indices, 0, this.indices.length);
- System.arraycopy(this.values, 0, values, 0, this.values.length);
-
- indices[this.indices.length] = n - 1;
- values[this.values.length] = d;
-
- return new SparseVector(n, indices, values);
- }
-
- /** Get the indices array. */
- public int[] getIndices() {
- return indices;
- }
-
- /** Get the values array. */
- public double[] getValues() {
- return values;
- }
-
- @Override
- public int size() {
- return n;
- }
-
- @Override
- public double get(int i) {
- int pos = Arrays.binarySearch(indices, i);
- if (pos >= 0) {
- return values[pos];
- }
- return 0.;
- }
-
- /** Set the size of the vector. */
- public void setSize(int n) {
- this.n = n;
- }
-
- /** Get number of values in this vector. */
- public int numberOfValues() {
- return this.values.length;
- }
-
- @Override
- public void set(int i, double val) {
- int pos = Arrays.binarySearch(indices, i);
- if (pos >= 0) {
- this.values[pos] = val;
- } else {
- pos = -(pos + 1);
- insert(pos, i, val);
- }
- }
-
- @Override
- public void add(int i, double val) {
- int pos = Arrays.binarySearch(indices, i);
- if (pos >= 0) {
- this.values[pos] += val;
- } else {
- pos = -(pos + 1);
- insert(pos, i, val);
- }
- }
-
- /** Insert value "val" in the position "pos" with index "index". */
- private void insert(int pos, int index, double val) {
- double[] newValues = new double[this.values.length + 1];
- int[] newIndices = new int[this.values.length + 1];
- System.arraycopy(this.values, 0, newValues, 0, pos);
- System.arraycopy(this.indices, 0, newIndices, 0, pos);
- newValues[pos] = val;
- newIndices[pos] = index;
- System.arraycopy(this.values, pos, newValues, pos + 1, this.values.length - pos);
- System.arraycopy(this.indices, pos, newIndices, pos + 1, this.values.length - pos);
- this.values = newValues;
- this.indices = newIndices;
- }
-
- @Override
- public String toString() {
- return VectorUtil.toString(this);
- }
-
- @Override
- public double normL2() {
- double d = 0;
- for (double t : values) {
- d += t * t;
- }
- return Math.sqrt(d);
- }
-
- @Override
- public double normL1() {
- double d = 0;
- for (double t : values) {
- d += Math.abs(t);
- }
- return d;
- }
-
- @Override
- public double normInf() {
- double d = 0;
- for (double t : values) {
- d = Math.max(Math.abs(t), d);
- }
- return d;
- }
-
- @Override
- public double normL2Square() {
- double d = 0;
- for (double t : values) {
- d += t * t;
- }
- return d;
- }
-
- @Override
- public SparseVector slice(int[] indices) {
- SparseVector sliced = new SparseVector(indices.length);
- int nnz = 0;
- sliced.indices = new int[indices.length];
- sliced.values = new double[indices.length];
-
- for (int i = 0; i < indices.length; i++) {
- int pos = Arrays.binarySearch(this.indices, indices[i]);
- if (pos >= 0) {
- sliced.indices[nnz] = i;
- sliced.values[nnz] = this.values[pos];
- nnz++;
- }
- }
-
- if (nnz < sliced.indices.length) {
- sliced.indices = Arrays.copyOf(sliced.indices, nnz);
- sliced.values = Arrays.copyOf(sliced.values, nnz);
- }
-
- return sliced;
- }
-
- @Override
- public Vector plus(Vector vec) {
- if (this.size() != vec.size()) {
- throw new IllegalArgumentException("The size of the two vectors are different.");
- }
-
- if (vec instanceof DenseVector) {
- DenseVector r = ((DenseVector) vec).clone();
- for (int i = 0; i < this.indices.length; i++) {
- r.add(this.indices[i], this.values[i]);
- }
- return r;
- } else {
- return MatVecOp.apply(this, (SparseVector) vec, ((a, b) -> a + b));
- }
- }
-
- @Override
- public Vector minus(Vector vec) {
- if (this.size() != vec.size()) {
- throw new IllegalArgumentException("The size of the two vectors are different.");
- }
-
- if (vec instanceof DenseVector) {
- DenseVector r = ((DenseVector) vec).scale(-1.0);
- for (int i = 0; i < this.indices.length; i++) {
- r.add(this.indices[i], this.values[i]);
- }
- return r;
- } else {
- return MatVecOp.apply(this, (SparseVector) vec, ((a, b) -> a - b));
- }
- }
-
- @Override
- public SparseVector scale(double d) {
- SparseVector r = this.clone();
- BLAS.scal(d, r);
- return r;
- }
-
- @Override
- public void scaleEqual(double d) {
- BLAS.scal(d, this);
- }
-
- /** Remove all zero values away from this vector. */
- public void removeZeroValues() {
- if (this.values.length != 0) {
- List<Integer> idxs = new ArrayList<>();
- for (int i = 0; i < values.length; i++) {
- if (0 != values[i]) {
- idxs.add(i);
- }
- }
- int[] newIndices = new int[idxs.size()];
- double[] newValues = new double[newIndices.length];
- for (int i = 0; i < newIndices.length; i++) {
- newIndices[i] = indices[idxs.get(i)];
- newValues[i] = values[idxs.get(i)];
- }
- this.indices = newIndices;
- this.values = newValues;
- }
- }
-
- private double dot(SparseVector other) {
- if (this.size() != other.size()) {
- throw new RuntimeException("the size of the two vectors are different");
- }
-
- double d = 0;
- int p0 = 0;
- int p1 = 0;
- while (p0 < this.values.length && p1 < other.values.length) {
- if (this.indices[p0] == other.indices[p1]) {
- d += this.values[p0] * other.values[p1];
- p0++;
- p1++;
- } else if (this.indices[p0] < other.indices[p1]) {
- p0++;
- } else {
- p1++;
- }
- }
- return d;
- }
-
- private double dot(DenseVector other) {
- if (this.size() != other.size()) {
- throw new RuntimeException(
- "The size of the two vectors are different: "
- + this.size()
- + " vs "
- + other.size());
- }
- double s = 0.;
- for (int i = 0; i < this.indices.length; i++) {
- s += this.values[i] * other.get(this.indices[i]);
- }
- return s;
- }
-
- @Override
- public double dot(Vector other) {
- if (other instanceof DenseVector) {
- return dot((DenseVector) other);
- } else {
- return dot((SparseVector) other);
- }
- }
-
- @Override
- public DenseMatrix outer() {
- return this.outer(this);
- }
-
- /**
- * Compute the outer product with another vector.
- *
- * @return The outer product matrix.
- */
- public DenseMatrix outer(SparseVector other) {
- int nrows = this.size();
- int ncols = other.size();
- double[] data = new double[ncols * nrows];
- for (int i = 0; i < this.values.length; i++) {
- for (int j = 0; j < other.values.length; j++) {
- data[this.indices[i] + other.indices[j] * nrows] = this.values[i] * other.values[j];
- }
- }
- return new DenseMatrix(nrows, ncols, data);
- }
-
- /** Convert to a dense vector. */
- public DenseVector toDenseVector() {
- if (n >= 0) {
- DenseVector r = new DenseVector(n);
- for (int i = 0; i < this.indices.length; i++) {
- r.set(this.indices[i], this.values[i]);
- }
- return r;
- } else {
- if (this.indices.length == 0) {
- return new DenseVector();
- } else {
- int n = this.indices[this.indices.length - 1] + 1;
- DenseVector r = new DenseVector(n);
- for (int i = 0; i < this.indices.length; i++) {
- r.set(this.indices[i], this.values[i]);
- }
- return r;
- }
- }
- }
-
- @Override
- public void standardizeEqual(double mean, double stdvar) {
- for (int i = 0; i < indices.length; i++) {
- values[i] -= mean;
- values[i] *= (1.0 / stdvar);
- }
- }
-
- @Override
- public void normalizeEqual(double p) {
- double norm = 0.0;
- if (Double.isInfinite(p)) {
- norm = normInf();
- } else if (p == 1.0) {
- norm = normL1();
- } else if (p == 2.0) {
- norm = normL2();
- } else {
- for (int i = 0; i < indices.length; i++) {
- norm += Math.pow(values[i], p);
- }
- norm = Math.pow(norm, 1 / p);
- }
-
- for (int i = 0; i < indices.length; i++) {
- values[i] /= norm;
- }
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- SparseVector that = (SparseVector) o;
- return n == that.n
- && Arrays.equals(indices, that.indices)
- && Arrays.equals(values, that.values);
- }
-
- @Override
- public int hashCode() {
- int result = Objects.hash(n);
- result = 31 * result + Arrays.hashCode(indices);
- result = 31 * result + Arrays.hashCode(values);
- return result;
- }
-
- @Override
- public VectorIterator iterator() {
- return new SparseVectorVectorIterator();
- }
-
- private class SparseVectorVectorIterator implements VectorIterator {
- private int cursor = 0;
-
- @Override
- public boolean hasNext() {
- return cursor < values.length;
- }
-
- @Override
- public void next() {
- cursor++;
- }
-
- @Override
- public int getIndex() {
- if (cursor >= values.length) {
- throw new RuntimeException("Iterator out of bound.");
- }
- return indices[cursor];
- }
-
- @Override
- public double getValue() {
- if (cursor >= values.length) {
- throw new RuntimeException("Iterator out of bound.");
- }
- return values[cursor];
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/Vector.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/Vector.java
deleted file mode 100644
index bf4e329..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/Vector.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import java.io.Serializable;
-
-/** The Vector class defines some common methods for both DenseVector and SparseVector. */
-public abstract class Vector implements Serializable {
- /** Get the size of the vector. */
- public abstract int size();
-
- /** Get the i-th element of the vector. */
- public abstract double get(int i);
-
- /** Set the i-th element of the vector to value "val". */
- public abstract void set(int i, double val);
-
- /** Add the i-th element of the vector by value "val". */
- public abstract void add(int i, double val);
-
- /** Return the L1 norm of the vector. */
- public abstract double normL1();
-
- /** Return the Inf norm of the vector. */
- public abstract double normInf();
-
- /** Return the L2 norm of the vector. */
- public abstract double normL2();
-
- /** Return the square of L2 norm of the vector. */
- public abstract double normL2Square();
-
- /** Scale the vector by value "v" and create a new vector to store the result. */
- public abstract Vector scale(double v);
-
- /** Scale the vector by value "v". */
- public abstract void scaleEqual(double v);
-
- /** Normalize the vector. */
- public abstract void normalizeEqual(double p);
-
- /** Standardize the vector. */
- public abstract void standardizeEqual(double mean, double stdvar);
-
- /** Create a new vector by adding an element to the head of the vector. */
- public abstract Vector prefix(double v);
-
- /** Create a new vector by adding an element to the end of the vector. */
- public abstract Vector append(double v);
-
- /** Create a new vector by plussing another vector. */
- public abstract Vector plus(Vector vec);
-
- /** Create a new vector by subtracting another vector. */
- public abstract Vector minus(Vector vec);
-
- /** Compute the dot product with another vector. */
- public abstract double dot(Vector vec);
-
- /** Get the iterator of the vector. */
- public abstract VectorIterator iterator();
-
- /** Slice the vector. */
- public abstract Vector slice(int[] indexes);
-
- /**
- * Compute the outer product with itself.
- *
- * @return The outer product matrix.
- */
- public abstract DenseMatrix outer();
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorIterator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorIterator.java
deleted file mode 100644
index 422b7a4..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorIterator.java
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import java.io.Serializable;
-import java.util.Iterator;
-
-/**
- * An iterator over the elements of a vector.
- *
- * <p>Usage: <code>
- * Vector vector = ...;
- * VectorIterator iterator = vector.iterator();
- *
- * while(iterator.hasNext()) {
- * int index = iterator.getIndex();
- * double value = iterator.getValue();
- * iterator.next();
- * }
- * </code>
- */
-public interface VectorIterator extends Serializable {
-
- /**
- * Returns {@code true} if the iteration has more elements. Otherwise, {@code false} will be
- * returned.
- *
- * @return {@code true} if the iteration has more elements
- */
- boolean hasNext();
-
- /**
- * Trigger the cursor points to the next element of the vector.
- *
- * <p>The {@link #getIndex()} while returns the index of the element which the cursor points.
- * The {@link #getValue()} ()} while returns the value of the element which the cursor points.
- *
- * <p>The difference to the {@link Iterator#next()} is that this can avoid the return of boxed
- * type.
- */
- void next();
-
- /**
- * Returns the index of the element which the cursor points.
- *
- * @returnthe the index of the element which the cursor points.
- */
- int getIndex();
-
- /**
- * Returns the value of the element which the cursor points.
- *
- * @returnthe the value of the element which the cursor points.
- */
- double getValue();
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorUtil.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorUtil.java
deleted file mode 100644
index 7e328ca..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/linalg/VectorUtil.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.apache.commons.lang3.StringUtils;
-
-/** Utility class for the operations on {@link Vector} and its subclasses. */
-public class VectorUtil {
- /** Delimiter between elements. */
- private static final char ELEMENT_DELIMITER = ' ';
- /** Delimiter between vector size and vector data. */
- private static final char HEADER_DELIMITER = '$';
- /** Delimiter between index and value. */
- private static final char INDEX_VALUE_DELIMITER = ':';
-
- /**
- * Parse either a {@link SparseVector} or a {@link DenseVector} from a formatted string.
- *
- * <p>The format of a dense vector is space separated values such as "1 2 3 4". The format of a
- * sparse vector is space separated index-value pairs, such as "0:1 2:3 3:4". If the sparse
- * vector has determined vector size, the size is prepended to the head. For example, the string
- * "$4$0:1 2:3 3:4" represents a sparse vector with size 4.
- *
- * @param str A formatted string representing a vector.
- * @return The parsed vector.
- */
- public static Vector parse(String str) {
- boolean isSparse =
- org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly(str)
- || StringUtils.indexOf(str, INDEX_VALUE_DELIMITER) != -1
- || StringUtils.indexOf(str, HEADER_DELIMITER) != -1;
- if (isSparse) {
- return parseSparse(str);
- } else {
- return parseDense(str);
- }
- }
-
- /**
- * Parse the dense vector from a formatted string.
- *
- * <p>The format of a dense vector is space separated values such as "1 2 3 4".
- *
- * @param str A string of space separated values.
- * @return The parsed vector.
- */
- public static DenseVector parseDense(String str) {
- if (org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly(str)) {
- return new DenseVector();
- }
-
- int len = str.length();
-
- int inDataBuffPos = 0;
- boolean isInBuff = false;
-
- for (int i = 0; i < len; ++i) {
- char c = str.charAt(i);
-
- if (c == ELEMENT_DELIMITER
- // to be compatible with previous delimiter
- || c == ',') {
- if (isInBuff) {
- inDataBuffPos++;
- }
-
- isInBuff = false;
- } else {
- isInBuff = true;
- }
- }
-
- if (isInBuff) {
- inDataBuffPos++;
- }
-
- double[] data = new double[inDataBuffPos];
- int lastestInCharBuffPos = 0;
-
- inDataBuffPos = 0;
- isInBuff = false;
-
- for (int i = 0; i < len; ++i) {
- char c = str.charAt(i);
-
- if (c == ELEMENT_DELIMITER) {
- if (isInBuff) {
- data[inDataBuffPos++] =
- Double.parseDouble(
- StringUtils.substring(str, lastestInCharBuffPos, i).trim());
-
- lastestInCharBuffPos = i + 1;
- }
-
- isInBuff = false;
- } else {
- isInBuff = true;
- }
- }
-
- if (isInBuff) {
- data[inDataBuffPos] =
- Double.valueOf(StringUtils.substring(str, lastestInCharBuffPos).trim());
- }
-
- return new DenseVector(data);
- }
-
- /**
- * Parse the sparse vector from a formatted string.
- *
- * <p>The format of a sparse vector is space separated index-value pairs, such as "0:1 2:3 3:4".
- * If the sparse vector has determined vector size, the size is prepended to the head. For
- * example, the string "$4$0:1 2:3 3:4" represents a sparse vector with size 4.
- *
- * @param str A formatted string representing a sparse vector.
- * @throws IllegalArgumentException If the string is of invalid format.
- */
- public static SparseVector parseSparse(String str) {
- try {
- if (org.apache.flink.util.StringUtils.isNullOrWhitespaceOnly(str)) {
- return new SparseVector();
- }
-
- int n = -1;
- int firstDollarPos = str.indexOf(HEADER_DELIMITER);
- int lastDollarPos = -1;
- if (firstDollarPos >= 0) {
- lastDollarPos = StringUtils.lastIndexOf(str, HEADER_DELIMITER);
- String sizeStr = StringUtils.substring(str, firstDollarPos + 1, lastDollarPos);
- n = Integer.valueOf(sizeStr);
- if (lastDollarPos == str.length() - 1) {
- return new SparseVector(n);
- }
- }
-
- int numValues = StringUtils.countMatches(str, String.valueOf(INDEX_VALUE_DELIMITER));
- double[] data = new double[numValues];
- int[] indices = new int[numValues];
- int startPos = lastDollarPos + 1;
- int endPos;
- for (int i = 0; i < numValues; i++) {
- int colonPos = StringUtils.indexOf(str, INDEX_VALUE_DELIMITER, startPos);
- if (colonPos < 0) {
- throw new IllegalArgumentException("Format error.");
- }
- endPos = StringUtils.indexOf(str, ELEMENT_DELIMITER, colonPos);
-
- if (endPos == -1) {
- endPos = str.length();
- }
- indices[i] = Integer.valueOf(str.substring(startPos, colonPos).trim());
- data[i] = Double.valueOf(str.substring(colonPos + 1, endPos).trim());
- startPos = endPos + 1;
- }
- return new SparseVector(n, indices, data);
- } catch (Exception e) {
- throw new IllegalArgumentException(
- String.format("Fail to getVector sparse vector from string: \"%s\".", str), e);
- }
- }
-
- /**
- * Serialize the vector to a string.
- *
- * @param vector The vector to serialize.
- * @see #toString(DenseVector)
- * @see #toString(SparseVector)
- */
- public static String toString(Vector vector) {
- if (vector instanceof SparseVector) {
- return toString((SparseVector) vector);
- }
- return toString((DenseVector) vector);
- }
-
- /**
- * Serialize the SparseVector to string.
- *
- * <p>The format of the returned is described at {@link #parseSparse(String)}
- *
- * @param sparseVector The sparse vector to serialize.
- */
- public static String toString(SparseVector sparseVector) {
- StringBuilder sbd = new StringBuilder();
- if (sparseVector.n > 0) {
- sbd.append(HEADER_DELIMITER);
- sbd.append(sparseVector.n);
- sbd.append(HEADER_DELIMITER);
- }
- if (null != sparseVector.indices) {
- for (int i = 0; i < sparseVector.indices.length; i++) {
- sbd.append(sparseVector.indices[i]);
- sbd.append(INDEX_VALUE_DELIMITER);
- sbd.append(sparseVector.values[i]);
- if (i < sparseVector.indices.length - 1) {
- sbd.append(ELEMENT_DELIMITER);
- }
- }
- }
-
- return sbd.toString();
- }
-
- /**
- * Serialize the DenseVector to String.
- *
- * <p>The format of the returned is described at {@link #parseDense(String)}
- *
- * @param denseVector The DenseVector to serialize.
- */
- public static String toString(DenseVector denseVector) {
- StringBuilder sbd = new StringBuilder();
-
- for (int i = 0; i < denseVector.data.length; i++) {
- sbd.append(denseVector.data[i]);
- if (i < denseVector.data.length - 1) {
- sbd.append(ELEMENT_DELIMITER);
- }
- }
- return sbd.toString();
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/Mapper.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/Mapper.java
deleted file mode 100644
index 4cce10b..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/Mapper.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.mapper;
-
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.types.DataType;
-import org.apache.flink.types.Row;
-
-import java.io.Serializable;
-
-/**
- * Abstract class for mappers. A mapper takes one row as input and transform it into another row.
- */
-public abstract class Mapper implements Serializable {
-
- /** Schema of the input rows. */
- private final String[] dataFieldNames;
-
- private final DataType[] dataFieldTypes;
-
- /** Parameters for the Mapper. Users can set the params before the Mapper is executed. */
- protected final Params params;
-
- /**
- * Construct a Mapper.
- *
- * @param dataSchema The schema of input rows.
- * @param params The parameters for this mapper.
- */
- public Mapper(TableSchema dataSchema, Params params) {
- this.dataFieldNames = dataSchema.getFieldNames();
- this.dataFieldTypes = dataSchema.getFieldDataTypes();
- this.params = (null == params) ? new Params() : params.clone();
- }
-
- /**
- * Get the schema of input rows.
- *
- * @return The schema of input rows.
- */
- protected TableSchema getDataSchema() {
- return TableSchema.builder().fields(dataFieldNames, dataFieldTypes).build();
- }
-
- /**
- * Map a row to a new row.
- *
- * @param row The input row.
- * @return A new row.
- * @throws Exception This method may throw exceptions. Throwing an exception will cause the
- * operation to fail.
- */
- public abstract Row map(Row row) throws Exception;
-
- /**
- * Get the schema of the output rows of {@link #map(Row)} method.
- *
- * @return The table schema of the output rows of {@link #map(Row)} method.
- */
- public abstract TableSchema getOutputSchema();
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/MapperAdapter.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/MapperAdapter.java
deleted file mode 100644
index d804302..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/MapperAdapter.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.mapper;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.types.Row;
-
-/**
- * A class that helps adapt a {@link Mapper} to a {@link MapFunction} so that the mapper can run in
- * Flink.
- */
-public class MapperAdapter implements MapFunction<Row, Row> {
-
- private final Mapper mapper;
-
- /**
- * Construct a MapperAdapter with the given mapper.
- *
- * @param mapper The {@link Mapper} to adapt.
- */
- public MapperAdapter(Mapper mapper) {
- this.mapper = mapper;
- }
-
- @Override
- public Row map(Row row) throws Exception {
- return this.mapper.map(row);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapper.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapper.java
deleted file mode 100644
index 772e6d6..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapper.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.mapper;
-
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.types.DataType;
-import org.apache.flink.types.Row;
-
-import java.util.List;
-
-/** An abstract class for {@link Mapper Mappers} with a model. */
-public abstract class ModelMapper extends Mapper {
-
- /** Field names of the model rows. */
- private final String[] modelFieldNames;
-
- /** Field types of the model rows. */
- private final DataType[] modelFieldTypes;
-
- /**
- * Constructs a ModelMapper.
- *
- * @param modelSchema The schema of the model rows passed to {@link #loadModel(List)}.
- * @param dataSchema The schema of the input data rows.
- * @param params The parameters of this ModelMapper.
- */
- public ModelMapper(TableSchema modelSchema, TableSchema dataSchema, Params params) {
- super(dataSchema, params);
- this.modelFieldNames = modelSchema.getFieldNames();
- this.modelFieldTypes = modelSchema.getFieldDataTypes();
- }
-
- /**
- * Get the schema of the model rows that are passed to {@link #loadModel(List)}.
- *
- * @return The schema of the model rows.
- */
- protected TableSchema getModelSchema() {
- return TableSchema.builder().fields(this.modelFieldNames, this.modelFieldTypes).build();
- }
-
- /**
- * Load the model from the list of rows.
- *
- * @param modelRows The list of rows that containing the model.
- */
- public abstract void loadModel(List<Row> modelRows);
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapperAdapter.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapperAdapter.java
deleted file mode 100644
index 821b383..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/mapper/ModelMapperAdapter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.mapper;
-
-import org.apache.flink.api.common.functions.RichMapFunction;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.ml.common.model.ModelSource;
-import org.apache.flink.types.Row;
-
-import java.util.List;
-
-/**
- * A class that adapts a {@link ModelMapper} to a Flink {@link RichMapFunction} so the model can be
- * loaded in a Flink job.
- *
- * <p>This adapter class hold the target {@link ModelMapper} and it's {@link ModelSource}. Upon
- * open(), it will load model rows from {@link ModelSource} into {@link ModelMapper}.
- */
-public class ModelMapperAdapter extends RichMapFunction<Row, Row> {
-
- private final ModelMapper mapper;
- private final ModelSource modelSource;
-
- /**
- * Construct a ModelMapperAdapter with the given ModelMapper and ModelSource.
- *
- * @param mapper The {@link ModelMapper} to adapt.
- * @param modelSource The {@link ModelSource} to load the model from.
- */
- public ModelMapperAdapter(ModelMapper mapper, ModelSource modelSource) {
- this.mapper = mapper;
- this.modelSource = modelSource;
- }
-
- @Override
- public void open(Configuration parameters) throws Exception {
- List<Row> modelRows = this.modelSource.getModelRows(getRuntimeContext());
- this.mapper.loadModel(modelRows);
- }
-
- @Override
- public Row map(Row row) throws Exception {
- return this.mapper.map(row);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/BroadcastVariableModelSource.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/BroadcastVariableModelSource.java
deleted file mode 100644
index 0e552ae..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/BroadcastVariableModelSource.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.model;
-
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.types.Row;
-
-import java.util.List;
-
-/** A {@link ModelSource} implementation that reads the model from the broadcast variable. */
-public class BroadcastVariableModelSource implements ModelSource {
-
- /** The name of the broadcast variable that hosts the model. */
- private final String modelVariableName;
-
- /**
- * Construct a BroadcastVariableModelSource.
- *
- * @param modelVariableName The name of the broadcast variable that hosts a
- * BroadcastVariableModelSource.
- */
- public BroadcastVariableModelSource(String modelVariableName) {
- this.modelVariableName = modelVariableName;
- }
-
- @Override
- public List<Row> getModelRows(RuntimeContext runtimeContext) {
- return runtimeContext.getBroadcastVariable(modelVariableName);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/ModelSource.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/ModelSource.java
deleted file mode 100644
index 960f9aa..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/ModelSource.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.model;
-
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.types.Row;
-
-import java.io.Serializable;
-import java.util.List;
-
-/**
- * An interface that load the model from different sources. E.g. broadcast variables, list of rows,
- * etc.
- */
-public interface ModelSource extends Serializable {
-
- /**
- * Get the rows that containing the model.
- *
- * @return the rows that containing the model.
- */
- List<Row> getModelRows(RuntimeContext runtimeContext);
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/RowsModelSource.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/RowsModelSource.java
deleted file mode 100644
index d314a70..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/model/RowsModelSource.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.model;
-
-import org.apache.flink.api.common.functions.RuntimeContext;
-import org.apache.flink.types.Row;
-
-import java.util.List;
-
-/** A {@link ModelSource} implementation that reads the model from the memory. */
-public class RowsModelSource implements ModelSource {
-
- /** The rows that hosts the model. */
- private final List<Row> modelRows;
-
- /**
- * Construct a RowsModelSource with the given rows containing a model.
- *
- * @param modelRows The rows that contains a model.
- */
- public RowsModelSource(List<Row> modelRows) {
- this.modelRows = modelRows;
- }
-
- @Override
- public List<Row> getModelRows(RuntimeContext runtimeContext) {
- return modelRows;
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussian.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussian.java
deleted file mode 100644
index 1f5f806..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussian.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.statistics.basicstatistic;
-
-import org.apache.flink.ml.common.linalg.BLAS;
-import org.apache.flink.ml.common.linalg.DenseMatrix;
-import org.apache.flink.ml.common.linalg.DenseVector;
-import org.apache.flink.ml.common.linalg.SparseVector;
-import org.apache.flink.ml.common.linalg.Vector;
-
-import org.apache.flink.shaded.guava18.com.google.common.primitives.Doubles;
-
-import com.github.fommil.netlib.LAPACK;
-import org.netlib.util.intW;
-
-/** This class provides basic functionality for a Multivariate Gaussian (Normal) Distribution. */
-public class MultivariateGaussian {
-
- private static final LAPACK LAPACK_INST = LAPACK.getInstance();
- private static final com.github.fommil.netlib.BLAS F2J_BLAS_INST =
- com.github.fommil.netlib.F2jBLAS.getInstance();
- private static final double EPSILON;
-
- static {
- double eps = 1.0;
- while ((1.0 + (eps / 2.0)) != 1.0) {
- eps /= 2.0;
- }
- EPSILON = eps;
- }
-
- private final DenseVector mean;
- private final DenseMatrix cov;
-
- private DenseMatrix rootSigmaInv;
- private double u;
-
- // data buffers for computing pdf
- private DenseVector delta;
- private DenseVector v;
-
- /**
- * The constructor.
- *
- * @param mean The mean vector of the distribution.
- * @param cov The covariance matrix of the distribution.
- */
- public MultivariateGaussian(DenseVector mean, DenseMatrix cov) {
- this.mean = mean;
- this.cov = cov;
- this.delta = DenseVector.zeros(mean.size());
- this.v = DenseVector.zeros(mean.size());
- calculateCovarianceConstants();
- }
-
- /** Returns density of this multivariate Gaussian at given point x . */
- public double pdf(Vector x) {
- return Math.exp(logpdf(x));
- }
-
- /** Returns the log-density of this multivariate Gaussian at given point x . */
- public double logpdf(Vector x) {
- int n = mean.size();
- System.arraycopy(mean.getData(), 0, delta.getData(), 0, n);
- BLAS.scal(-1.0, delta);
- if (x instanceof DenseVector) {
- BLAS.axpy(1., (DenseVector) x, delta);
- } else if (x instanceof SparseVector) {
- BLAS.axpy(1., (SparseVector) x, delta);
- }
- BLAS.gemv(1.0, rootSigmaInv, true, delta, 0., v);
- return u - 0.5 * BLAS.dot(v, v);
- }
-
- /**
- * Compute distribution dependent constants.
- *
- * <p>The probability density function is calculated as: pdf(x) = (2*pi)^(-k/2)^ *
- * det(sigma)^(-1/2)^ * exp((-1/2) * (x-mu).t * inv(sigma) * (x-mu))
- *
- * <p>Here we compute the following distribution dependent constants that can be reused in each
- * pdf computation: A) u = log((2*pi)^(-k/2)^ * det(sigma)^(-1/2)^) B) rootSigmaInv =
- * sqrt(inv(sigma)) = U * D^(-1/2)^
- *
- * <ul>
- * <li>sigma = U * D * U.t
- * <li>inv(sigma) = U * inv(D) * U.t = (U * D^(-1/2)^) * (U * D^(-1/2)^).t
- * <li>sqrt(inv(sigma)) = U * D^(-1/2)^
- * </ul>
- */
- private void calculateCovarianceConstants() {
- int k = this.mean.size();
- int lwork = 3 * k - 1;
- double[] matU = new double[k * k];
- double[] work = new double[lwork];
- double[] evs = new double[k];
- intW info = new intW(0);
-
- System.arraycopy(cov.getData(), 0, matU, 0, k * k);
- LAPACK_INST.dsyev("V", "U", k, matU, k, evs, work, lwork, info);
-
- double maxEv = Doubles.max(evs);
- double tol = EPSILON * k * maxEv;
-
- // log(pseudo-determinant) is sum of the logs of all non-zero singular values
- double logPseudoDetSigma = 0.;
- for (double ev : evs) {
- if (ev > tol) {
- logPseudoDetSigma += Math.log(ev);
- }
- }
-
- for (int i = 0; i < k; i++) {
- double invEv = evs[i] > tol ? Math.sqrt(1.0 / evs[i]) : 0.;
- F2J_BLAS_INST.dscal(k, invEv, matU, i * k, 1);
- }
- this.rootSigmaInv = new DenseMatrix(k, k, matU);
- this.u = -0.5 * (k * Math.log(2.0 * Math.PI) + logPseudoDetSigma);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataSetConversionUtil.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataSetConversionUtil.java
deleted file mode 100644
index f146a9f..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataSetConversionUtil.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.operators.SingleInputUdfOperator;
-import org.apache.flink.api.java.operators.TwoInputUdfOperator;
-import org.apache.flink.api.java.typeutils.RowTypeInfo;
-import org.apache.flink.ml.common.MLEnvironment;
-import org.apache.flink.ml.common.MLEnvironmentFactory;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.types.Row;
-
-/** Provide functions of conversions between DataSet and Table. */
-public class DataSetConversionUtil {
- /**
- * Convert the given Table to {@link DataSet}<{@link Row}>.
- *
- * @param sessionId the sessionId of {@link MLEnvironmentFactory}
- * @param table the Table to convert.
- * @return the converted DataSet.
- */
- public static DataSet<Row> fromTable(Long sessionId, Table table) {
- return MLEnvironmentFactory.get(sessionId)
- .getBatchTableEnvironment()
- .toDataSet(table, Row.class);
- }
-
- /**
- * Convert the given DataSet into a Table with specified TableSchema.
- *
- * @param sessionId the sessionId of {@link MLEnvironmentFactory}
- * @param data the DataSet to convert.
- * @param schema the specified TableSchema.
- * @return the converted Table.
- */
- public static Table toTable(Long sessionId, DataSet<Row> data, TableSchema schema) {
- // TableSchema.getFieldTypes() is deprecated, this should be improved once FLIP-65 is fully
- // merged.
- return toTable(sessionId, data, schema.getFieldNames(), schema.getFieldTypes());
- }
-
- /**
- * Convert the given DataSet into a Table with specified colNames and colTypes.
- *
- * @param sessionId sessionId the sessionId of {@link MLEnvironmentFactory}.
- * @param data the DataSet to convert.
- * @param colNames the specified colNames.
- * @param colTypes the specified colTypes. This variable is used only when the DataSet is
- * produced by a function and Flink cannot determine automatically what the produced type
- * is.
- * @return the converted Table.
- */
- public static Table toTable(
- Long sessionId, DataSet<Row> data, String[] colNames, TypeInformation<?>[] colTypes) {
- return toTable(MLEnvironmentFactory.get(sessionId), data, colNames, colTypes);
- }
-
- /**
- * Convert the given DataSet into a Table with specified colNames.
- *
- * @param sessionId sessionId the sessionId of {@link MLEnvironmentFactory}.
- * @param data the DataSet to convert.
- * @param colNames the specified colNames.
- * @return the converted Table.
- */
- public static Table toTable(Long sessionId, DataSet<Row> data, String[] colNames) {
- return toTable(MLEnvironmentFactory.get(sessionId), data, colNames);
- }
-
- /**
- * Convert the given DataSet into a Table with specified colNames and colTypes.
- *
- * @param session the MLEnvironment using to convert DataSet to Table.
- * @param data the DataSet to convert.
- * @param colNames the specified colNames.
- * @param colTypes the specified colTypes. This variable is used only when the DataSet is
- * produced by a function and Flink cannot determine automatically what the produced type
- * is.
- * @return the converted Table.
- */
- public static Table toTable(
- MLEnvironment session,
- DataSet<Row> data,
- String[] colNames,
- TypeInformation<?>[] colTypes) {
- try {
- if (null != colTypes) {
- // Try to add row type information for the dataset to be converted.
- // In most case, this keeps us from the rolling back logic in the catch block,
- // which adds an unnecessary map function just in order to add row type information.
- if (data instanceof SingleInputUdfOperator) {
- ((SingleInputUdfOperator) data).returns(new RowTypeInfo(colTypes, colNames));
- } else if (data instanceof TwoInputUdfOperator) {
- ((TwoInputUdfOperator) data).returns(new RowTypeInfo(colTypes, colNames));
- }
- }
- return toTable(session, data, colNames);
- } catch (ValidationException ex) {
- // currently ValidationException will be thrown and caught for further processing.
- // Because the getType() API of the Transformation can only be accessed once.
- // This can be improve if we add "isTypeSet()" API to the Transformation class.
- if (null == colTypes) {
- throw ex;
- } else {
- DataSet<Row> t = fallbackToExplicitTypeDefine(data, colNames, colTypes);
- return toTable(session, t, colNames);
- }
- }
- }
-
- /**
- * Convert the given DataSet into a Table with specified colNames.
- *
- * @param session the MLEnvironment using to convert DataSet to Table.
- * @param data the DataSet to convert.
- * @param colNames the specified colNames.
- * @return the converted Table.
- */
- public static Table toTable(MLEnvironment session, DataSet<Row> data, String[] colNames) {
- if (null == colNames || colNames.length == 0) {
- return session.getBatchTableEnvironment().fromDataSet(data);
- } else {
- StringBuilder sbd = new StringBuilder();
- sbd.append(colNames[0]);
- for (int i = 1; i < colNames.length; i++) {
- sbd.append(",").append(colNames[i]);
- }
- return session.getBatchTableEnvironment().fromDataSet(data, sbd.toString());
- }
- }
-
- /**
- * Adds a type information hint about the colTypes with the Row to the DataSet.
- *
- * @param data the DataSet to add type information.
- * @param colNames the specified colNames
- * @param colTypes the specified colTypes
- * @return the DataSet with type information hint.
- */
- private static DataSet<Row> fallbackToExplicitTypeDefine(
- DataSet<Row> data, String[] colNames, TypeInformation<?>[] colTypes) {
- return data.map((MapFunction<Row, Row>) t -> t)
- .returns(new RowTypeInfo(colTypes, colNames));
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataStreamConversionUtil.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataStreamConversionUtil.java
deleted file mode 100644
index 112e249..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/DataStreamConversionUtil.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.typeutils.RowTypeInfo;
-import org.apache.flink.ml.common.MLEnvironment;
-import org.apache.flink.ml.common.MLEnvironmentFactory;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.SingleOutputStreamOperator;
-import org.apache.flink.table.api.Expressions;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.table.expressions.Expression;
-import org.apache.flink.types.Row;
-
-import java.util.Arrays;
-
-/** Provide functions of conversions between DataStream and Table. */
-public class DataStreamConversionUtil {
- /**
- * Convert the given Table to {@link DataStream}<{@link Row}>.
- *
- * @param sessionId the sessionId of {@link MLEnvironmentFactory}
- * @param table the Table to convert.
- * @return the converted DataStream.
- */
- public static DataStream<Row> fromTable(Long sessionId, Table table) {
- return MLEnvironmentFactory.get(sessionId)
- .getStreamTableEnvironment()
- .toAppendStream(table, Row.class);
- }
-
- /**
- * Convert the given DataStream to Table with specified TableSchema.
- *
- * @param sessionId the sessionId of {@link MLEnvironmentFactory}
- * @param data the DataStream to convert.
- * @param schema the specified TableSchema.
- * @return the converted Table.
- */
- public static Table toTable(Long sessionId, DataStream<Row> data, TableSchema schema) {
- // TableSchema.getFieldTypes() is deprecated, this should be improved once FLIP-65 is fully
- // merged.
- return toTable(sessionId, data, schema.getFieldNames(), schema.getFieldTypes());
- }
-
- /**
- * Convert the given DataStream to Table with specified colNames.
- *
- * @param sessionId sessionId the sessionId of {@link MLEnvironmentFactory}.
- * @param data the DataStream to convert.
- * @param colNames the specified colNames.
- * @return the converted Table.
- */
- public static Table toTable(Long sessionId, DataStream<Row> data, String[] colNames) {
- return toTable(MLEnvironmentFactory.get(sessionId), data, colNames);
- }
-
- /**
- * Convert the given DataStream to Table with specified colNames and colTypes.
- *
- * @param sessionId sessionId the sessionId of {@link MLEnvironmentFactory}.
- * @param data the DataStream to convert.
- * @param colNames the specified colNames.
- * @param colTypes the specified colTypes. This variable is used only when the DataStream is
- * produced by a function and Flink cannot determine automatically what the produced type
- * is.
- * @return the converted Table.
- */
- public static Table toTable(
- Long sessionId,
- DataStream<Row> data,
- String[] colNames,
- TypeInformation<?>[] colTypes) {
- return toTable(MLEnvironmentFactory.get(sessionId), data, colNames, colTypes);
- }
-
- /**
- * Convert the given DataStream to Table with specified colNames.
- *
- * @param session the MLEnvironment using to convert DataStream to Table.
- * @param data the DataStream to convert.
- * @param colNames the specified colNames.
- * @return the converted Table.
- */
- public static Table toTable(MLEnvironment session, DataStream<Row> data, String[] colNames) {
- if (null == colNames || colNames.length == 0) {
- return session.getStreamTableEnvironment().fromDataStream(data);
- } else {
- return session.getStreamTableEnvironment()
- .fromDataStream(
- data,
- Arrays.stream(colNames).map(Expressions::$).toArray(Expression[]::new));
- }
- }
-
- /**
- * Convert the given DataStream to Table with specified colNames and colTypes.
- *
- * @param session the MLEnvironment using to convert DataStream to Table.
- * @param data the DataStream to convert.
- * @param colNames the specified colNames.
- * @param colTypes the specified colTypes. This variable is used only when the DataStream is
- * produced by a function and Flink cannot determine automatically what the produced type
- * is.
- * @return the converted Table.
- */
- public static Table toTable(
- MLEnvironment session,
- DataStream<Row> data,
- String[] colNames,
- TypeInformation<?>[] colTypes) {
- try {
- if (null != colTypes) {
- // Try to add row type information for the datastream to be converted.
- // In most case, this keeps us from the rolling back logic in the catch block,
- // which adds an unnecessary map function just in order to add row type information.
- if (data instanceof SingleOutputStreamOperator) {
- ((SingleOutputStreamOperator) data)
- .returns(new RowTypeInfo(colTypes, colNames));
- }
- }
- return toTable(session, data, colNames);
- } catch (ValidationException ex) {
- if (null == colTypes) {
- throw ex;
- } else {
- DataStream<Row> t = fallbackToExplicitTypeDefine(data, colNames, colTypes);
- return toTable(session, t, colNames);
- }
- }
- }
-
- /**
- * Adds a type information hint about the colTypes with the Row to the DataStream.
- *
- * @param data the DataStream to add type information.
- * @param colNames the specified colNames
- * @param colTypes the specified colTypes
- * @return the DataStream with type information hint.
- */
- private static DataStream<Row> fallbackToExplicitTypeDefine(
- DataStream<Row> data, String[] colNames, TypeInformation<?>[] colTypes) {
- return data.map((MapFunction<Row, Row>) t -> t)
- .returns(new RowTypeInfo(colTypes, colNames));
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/OutputColsHelper.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/OutputColsHelper.java
deleted file mode 100644
index 18ca429..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/OutputColsHelper.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.types.Row;
-
-import org.apache.commons.lang3.ArrayUtils;
-
-import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashSet;
-
-/**
- * Utils for merging input data with output data.
- *
- * <p>Input: 1) Schema of input data being predicted or transformed. 2) Output column names of the
- * prediction/transformation operator. 3) Output column types of the prediction/transformation
- * operator. 4) Reserved column names, which is a subset of input data's column names that we want
- * to preserve.
- *
- * <p>Output: 1)The result data schema. The result data is a combination of the preserved columns
- * and the operator's output columns.
- *
- * <p>Several rules are followed:
- *
- * <ul>
- * <li>If reserved columns are not given, then all columns of input data is reserved.
- * <li>The reserved columns are arranged ahead of the operator's output columns in the final
- * output.
- * <li>If some of the reserved column names overlap with those of operator's output columns, then
- * the operator's output columns override the conflicting reserved columns.
- * <li>The reserved columns in the result table preserve their orders as in the input table.
- * </ul>
- *
- * <p>For example, if we have input data schema of ["id":INT, "f1":FLOAT, "f2":DOUBLE], and the
- * operator outputs a column "label" with type STRING, and we want to preserve the column "id", then
- * we get the result schema of ["id":INT, "label":STRING].
- *
- * <p>end user should not directly interact with this helper class. instead it will be indirectly
- * used via concrete algorithms.
- */
-public class OutputColsHelper implements Serializable {
- private String[] inputColNames;
- private TypeInformation<?>[] inputColTypes;
- private String[] outputColNames;
- private TypeInformation<?>[] outputColTypes;
-
- /** Column indices in the input data that would be forward to the result. */
- private int[] reservedCols;
-
- /** The positions of reserved columns in the result. */
- private int[] reservedColsPosInResult;
-
- /** The positions of output columns in the result. */
- private int[] outputColsPosInResult;
-
- public OutputColsHelper(
- TableSchema inputSchema, String outputColName, TypeInformation<?> outputColType) {
- this(inputSchema, outputColName, outputColType, inputSchema.getFieldNames());
- }
-
- public OutputColsHelper(
- TableSchema inputSchema,
- String outputColName,
- TypeInformation<?> outputColType,
- String[] reservedColNames) {
- this(
- inputSchema,
- new String[] {outputColName},
- new TypeInformation<?>[] {outputColType},
- reservedColNames);
- }
-
- public OutputColsHelper(
- TableSchema inputSchema, String[] outputColNames, TypeInformation<?>[] outputColTypes) {
- this(inputSchema, outputColNames, outputColTypes, inputSchema.getFieldNames());
- }
-
- /**
- * The constructor.
- *
- * @param inputSchema Schema of input data being predicted or transformed.
- * @param outputColNames Output column names of the prediction/transformation operator.
- * @param outputColTypes Output column types of the prediction/transformation operator.
- * @param reservedColNames Reserved column names, which is a subset of input data's column names
- * that we want to preserve.
- */
- public OutputColsHelper(
- TableSchema inputSchema,
- String[] outputColNames,
- TypeInformation<?>[] outputColTypes,
- String[] reservedColNames) {
- this.inputColNames = inputSchema.getFieldNames();
- this.inputColTypes = inputSchema.getFieldTypes();
- this.outputColNames = outputColNames;
- this.outputColTypes = outputColTypes;
-
- HashSet<String> toReservedCols =
- new HashSet<>(
- Arrays.asList(
- reservedColNames == null ? this.inputColNames : reservedColNames));
- // the indices of the columns which need to be reserved.
- ArrayList<Integer> reservedColIndices = new ArrayList<>(toReservedCols.size());
- ArrayList<Integer> reservedColToResultIndex = new ArrayList<>(toReservedCols.size());
- outputColsPosInResult = new int[outputColNames.length];
- Arrays.fill(outputColsPosInResult, -1);
- int index = 0;
- for (int i = 0; i < inputColNames.length; i++) {
- int key = ArrayUtils.indexOf(outputColNames, inputColNames[i]);
- if (key >= 0) {
- outputColsPosInResult[key] = index++;
- continue;
- }
- // add these interested column.
- if (toReservedCols.contains(inputColNames[i])) {
- reservedColIndices.add(i);
- reservedColToResultIndex.add(index++);
- }
- }
- for (int i = 0; i < outputColsPosInResult.length; i++) {
- if (outputColsPosInResult[i] == -1) {
- outputColsPosInResult[i] = index++;
- }
- }
- // write reversed column information in array.
- this.reservedCols = new int[reservedColIndices.size()];
- this.reservedColsPosInResult = new int[reservedColIndices.size()];
- for (int i = 0; i < this.reservedCols.length; i++) {
- this.reservedCols[i] = reservedColIndices.get(i);
- this.reservedColsPosInResult[i] = reservedColToResultIndex.get(i);
- }
- }
-
- /**
- * Get the reserved columns' names.
- *
- * @return the reserved colNames.
- */
- public String[] getReservedColumns() {
- String[] passThroughColNames = new String[reservedCols.length];
- for (int i = 0; i < reservedCols.length; i++) {
- passThroughColNames[i] = inputColNames[reservedCols[i]];
- }
- return passThroughColNames;
- }
-
- /**
- * Get the result table schema. The result data is a combination of the preserved columns and
- * the operator's output columns.
- *
- * @return The result table schema.
- */
- public TableSchema getResultSchema() {
- int resultLength = reservedCols.length + outputColNames.length;
- String[] resultColNames = new String[resultLength];
- TypeInformation<?>[] resultColTypes = new TypeInformation[resultLength];
- for (int i = 0; i < reservedCols.length; i++) {
- resultColNames[reservedColsPosInResult[i]] = inputColNames[reservedCols[i]];
- resultColTypes[reservedColsPosInResult[i]] = inputColTypes[reservedCols[i]];
- }
- for (int i = 0; i < outputColsPosInResult.length; i++) {
- resultColNames[outputColsPosInResult[i]] = outputColNames[i];
- resultColTypes[outputColsPosInResult[i]] = outputColTypes[i];
- }
- return new TableSchema(resultColNames, resultColTypes);
- }
-
- /**
- * Merge the input row and the output row.
- *
- * @param input The input row being predicted or transformed.
- * @param output The output row of the prediction/transformation operator.
- * @return The result row, which is a combination of preserved columns and the operator's output
- * columns.
- */
- public Row getResultRow(Row input, Row output) {
- int numOutputs = outputColsPosInResult.length;
- if (output.getArity() != numOutputs) {
- throw new IllegalArgumentException("Invalid output size");
- }
- int resultLength = reservedCols.length + outputColNames.length;
- Row result = new Row(resultLength);
- for (int i = 0; i < reservedCols.length; i++) {
- result.setField(reservedColsPosInResult[i], input.getField(reservedCols[i]));
- }
- for (int i = 0; i < numOutputs; i++) {
- result.setField(outputColsPosInResult[i], output.getField(i));
- }
- return result;
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/TableUtil.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/TableUtil.java
deleted file mode 100644
index 0cfd396..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/TableUtil.java
+++ /dev/null
@@ -1,439 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.types.Row;
-import org.apache.flink.util.Preconditions;
-
-import org.apache.flink.shaded.guava18.com.google.common.base.Joiner;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.UUID;
-
-/** Utility to operator to interact with Table contents, such as rows and columns. */
-public class TableUtil {
- /**
- * Return a temp table named with prefix `temp_`, follow by a random UUID.
- *
- * <p>UUID hyphens ("-") will be replaced by underscores ("_").
- *
- * @return tableName
- */
- public static synchronized String getTempTableName() {
- return ("temp_" + UUID.randomUUID().toString().replaceAll("-", "_")).toLowerCase();
- }
-
- /**
- * Find the index of <code>targetCol</code> in string array <code>tableCols</code>. It will
- * ignore the case of the tableCols.
- *
- * @param tableCols a string array among which to find the targetCol.
- * @param targetCol the targetCol to find.
- * @return the index of the targetCol, if not found, returns -1.
- */
- public static int findColIndex(String[] tableCols, String targetCol) {
- Preconditions.checkNotNull(targetCol, "targetCol is null!");
- for (int i = 0; i < tableCols.length; i++) {
- if (targetCol.equalsIgnoreCase(tableCols[i])) {
- return i;
- }
- }
- return -1;
- }
-
- /**
- * Find the index of <code>targetCol</code> from the <code>tableSchema</code>.
- *
- * @param tableSchema the TableSchema among which to find the targetCol.
- * @param targetCol the targetCols to find.
- * @return the index of the targetCol.
- */
- public static int findColIndex(TableSchema tableSchema, String targetCol) {
- return findColIndex(tableSchema.getFieldNames(), targetCol);
- }
-
- /**
- * Find the indices of <code>targetCols</code> in string array <code>tableCols</code>. If <code>
- * targetCols</code> is null, it will be replaced by the <code>tableCols</code>
- *
- * @param tableCols a string array among which to find the targetCols.
- * @param targetCols the targetCols to find.
- * @return the indices of the targetCols.
- */
- public static int[] findColIndices(String[] tableCols, String[] targetCols) {
- if (targetCols == null) {
- int[] indices = new int[tableCols.length];
- for (int i = 0; i < tableCols.length; i++) {
- indices[i] = i;
- }
- return indices;
- }
- int[] indices = new int[targetCols.length];
- for (int i = 0; i < indices.length; i++) {
- indices[i] = findColIndex(tableCols, targetCols[i]);
- }
- return indices;
- }
-
- /**
- * Find the indices of <code>targetCols</code> from the <code>tableSchema</code>.
- *
- * @param tableSchema the TableSchema among which to find the targetCols.
- * @param targetCols the targetCols to find.
- * @return the indices of the targetCols.
- */
- public static int[] findColIndices(TableSchema tableSchema, String[] targetCols) {
- return findColIndices(tableSchema.getFieldNames(), targetCols);
- }
-
- /**
- * Find the types of the <code>targetCols</code>. If the targetCol not exist, return null.
- *
- * @param tableSchema TableSchema.
- * @param targetCols the targetCols to find.
- * @return the corresponding types.
- */
- public static TypeInformation<?>[] findColTypes(TableSchema tableSchema, String[] targetCols) {
- if (targetCols == null) {
- return tableSchema.getFieldTypes();
- }
- TypeInformation<?>[] types = new TypeInformation[targetCols.length];
- for (int i = 0; i < types.length; i++) {
- types[i] = findColType(tableSchema, targetCols[i]);
- }
- return types;
- }
-
- /**
- * Find the type of the <code>targetCol</code>. If the targetCol not exist, return null.
- *
- * @param tableSchema TableSchema
- * @param targetCol the targetCol to find.
- * @return the corresponding type.
- */
- public static TypeInformation<?> findColType(TableSchema tableSchema, String targetCol) {
- int index = findColIndex(tableSchema.getFieldNames(), targetCol);
-
- return index == -1 ? null : tableSchema.getFieldTypes()[index];
- }
-
- /**
- * Determine whether it is number type, number type includes double, long, byte, int, float and
- * short.
- *
- * @param dataType the dataType to determine.
- * @return whether it is number type
- */
- public static boolean isSupportedNumericType(TypeInformation<?> dataType) {
- return Types.DOUBLE == dataType
- || Types.LONG == dataType
- || Types.BYTE == dataType
- || Types.INT == dataType
- || Types.FLOAT == dataType
- || Types.SHORT == dataType;
- }
-
- /**
- * Determine whether it is a string type.
- *
- * @param dataType the dataType to determine.
- * @return whether it is string type
- */
- public static boolean isString(TypeInformation<?> dataType) {
- return Types.STRING == dataType;
- }
-
- /**
- * Determine whether it is a vector type.
- *
- * @param dataType the dataType to determine.
- * @return whether it is vector type
- */
- public static boolean isVector(TypeInformation<?> dataType) {
- return VectorTypes.VECTOR.equals(dataType)
- || VectorTypes.DENSE_VECTOR.equals(dataType)
- || VectorTypes.SPARSE_VECTOR.equals(dataType);
- }
-
- /**
- * Check whether <code>selectedCols</code> exist or not, if not exist, throw exception.
- *
- * @param tableCols a string array among which to find the target selectedCols.
- * @param selectedCols the selectedCols to assert.
- */
- public static void assertSelectedColExist(String[] tableCols, String... selectedCols) {
- if (null != selectedCols) {
- for (String selectedCol : selectedCols) {
- if (null != selectedCol) {
- if (-1 == findColIndex(tableCols, selectedCol)) {
- throw new IllegalArgumentException(" col is not exist " + selectedCol);
- }
- }
- }
- }
- }
-
- /**
- * Check whether colTypes of the <code>selectedCols</code> is numerical, if not, throw
- * exception.
- *
- * @param tableSchema TableSchema
- * @param selectedCols the selectedCols to assert.
- */
- public static void assertNumericalCols(TableSchema tableSchema, String... selectedCols) {
- if (selectedCols != null && selectedCols.length != 0) {
- for (String selectedCol : selectedCols) {
- if (null != selectedCol) {
- if (!isSupportedNumericType(findColType(tableSchema, selectedCol))) {
- throw new IllegalArgumentException(
- "col type must be number " + selectedCol);
- }
- }
- }
- }
- }
-
- /**
- * Check whether colTypes of the <code>selectedCols</code> is string, if not, throw exception.
- *
- * @param tableSchema TableSchema
- * @param selectedCols the selectedCol to assert.
- */
- public static void assertStringCols(TableSchema tableSchema, String... selectedCols) {
- if (selectedCols != null && selectedCols.length != 0) {
- for (String selectedCol : selectedCols) {
- if (null != selectedCol) {
- if (!isString(findColType(tableSchema, selectedCol))) {
- throw new IllegalArgumentException(
- "col type must be string " + selectedCol);
- }
- }
- }
- }
- }
-
- /**
- * Check whether colTypes of the <code>selectedCols</code> is vector, if not, throw exception.
- *
- * @param tableSchema TableSchema
- * @param selectedCols the selectedCol to assert.
- * @see #isVector(TypeInformation)
- */
- public static void assertVectorCols(TableSchema tableSchema, String... selectedCols) {
- if (selectedCols != null && selectedCols.length != 0) {
- for (String selectedCol : selectedCols) {
- if (null != selectedCol) {
- if (!isVector(findColType(tableSchema, selectedCol))) {
- throw new IllegalArgumentException(
- "col type must be string " + selectedCol);
- }
- }
- }
- }
- }
-
- /**
- * Return the columns in the table whose types are string.
- *
- * @param tableSchema TableSchema
- * @return String columns.
- */
- public static String[] getStringCols(TableSchema tableSchema) {
- return getStringCols(tableSchema, null);
- }
-
- /**
- * Return the columns in the table whose types are string and are not included in the
- * excludeCols.
- *
- * <p>If <code>excludeCols</code> is null, return all the string columns.
- *
- * @param tableSchema TableSchema.
- * @param excludeCols The columns who are not considered.
- * @return string columns.
- */
- public static String[] getStringCols(TableSchema tableSchema, String[] excludeCols) {
- ArrayList<String> numericCols = new ArrayList<>();
- List<String> excludeColsList = null == excludeCols ? null : Arrays.asList(excludeCols);
- String[] inColNames = tableSchema.getFieldNames();
- TypeInformation<?>[] inColTypes = tableSchema.getFieldTypes();
-
- for (int i = 0; i < inColNames.length; i++) {
- if (isString(inColTypes[i])) {
- if (null == excludeColsList || !excludeColsList.contains(inColNames[i])) {
- numericCols.add(inColNames[i]);
- }
- }
- }
-
- return numericCols.toArray(new String[0]);
- }
-
- /**
- * Return the columns in the table whose types are numeric.
- *
- * @param tableSchema TableSchema
- * @return numeric columns.
- */
- public static String[] getNumericCols(TableSchema tableSchema) {
- return getNumericCols(tableSchema, null);
- }
-
- /**
- * Return the columns in the table whose types are numeric and are not included in the
- * excludeCols.
- *
- * <p>If <code>excludeCols</code> is null, return all the numeric columns.
- *
- * @param tableSchema TableSchema.
- * @param excludeCols the columns who are not considered.
- * @return numeric columns.
- */
- public static String[] getNumericCols(TableSchema tableSchema, String[] excludeCols) {
- ArrayList<String> numericCols = new ArrayList<>();
- List<String> excludeColsList = (null == excludeCols ? null : Arrays.asList(excludeCols));
- String[] inColNames = tableSchema.getFieldNames();
- TypeInformation<?>[] inColTypes = tableSchema.getFieldTypes();
-
- for (int i = 0; i < inColNames.length; i++) {
- if (isSupportedNumericType(inColTypes[i])) {
- if (null == excludeColsList || !excludeColsList.contains(inColNames[i])) {
- numericCols.add(inColNames[i]);
- }
- }
- }
-
- return numericCols.toArray(new String[0]);
- }
-
- /**
- * Get the columns from featureCols who are included in the <code>categoricalCols</code>, and
- * the columns whose types are string or boolean.
- *
- * <p>If <code>categoricalCols</code> is null, return all the categorical columns.
- *
- * <p>for example: In FeatureHasher which projects a number of categorical or numerical features
- * into a feature vector of a specified dimension needs to identify the categorical features.
- * And the column which is the string or boolean must be categorical. We need to find these
- * columns as categorical when user do not specify the types(categorical or numerical).
- *
- * @param tableSchema TableSchema.
- * @param featureCols the columns to chosen from.
- * @param categoricalCols the columns which are included in the final result whatever the types
- * of them are. And it must be a subset of featureCols.
- * @return the categoricalCols.
- */
- public static String[] getCategoricalCols(
- TableSchema tableSchema, String[] featureCols, String[] categoricalCols) {
- if (null == featureCols) {
- return categoricalCols;
- }
- List<String> categoricalList =
- null == categoricalCols ? null : Arrays.asList(categoricalCols);
- List<String> featureList = Arrays.asList(featureCols);
- if (null != categoricalCols && !featureList.containsAll(categoricalList)) {
- throw new IllegalArgumentException("CategoricalCols must be included in featureCols!");
- }
-
- TypeInformation<?>[] featureColTypes = findColTypes(tableSchema, featureCols);
- List<String> res = new ArrayList<>();
- for (int i = 0; i < featureCols.length; i++) {
- boolean included = null != categoricalList && categoricalList.contains(featureCols[i]);
- if (included
- || Types.BOOLEAN == featureColTypes[i]
- || Types.STRING == featureColTypes[i]) {
- res.add(featureCols[i]);
- }
- }
-
- return res.toArray(new String[0]);
- }
-
- /** format the column names as header of markdown. */
- public static String formatTitle(String[] colNames) {
- StringBuilder sbd = new StringBuilder();
- StringBuilder sbdSplitter = new StringBuilder();
-
- for (int i = 0; i < colNames.length; ++i) {
- if (i > 0) {
- sbd.append("|");
- sbdSplitter.append("|");
- }
-
- sbd.append(colNames[i]);
-
- int t = null == colNames[i] ? 4 : colNames[i].length();
- for (int j = 0; j < t; j++) {
- sbdSplitter.append("-");
- }
- }
-
- return sbd.toString() + "\r\n" + sbdSplitter.toString();
- }
-
- /** format the row as body of markdown. */
- public static String formatRows(Row row) {
- StringBuilder sbd = new StringBuilder();
-
- for (int i = 0; i < row.getArity(); ++i) {
- if (i > 0) {
- sbd.append("|");
- }
- Object obj = row.getField(i);
- if (obj instanceof Double || obj instanceof Float) {
- sbd.append(String.format("%.4f", (double) obj));
- } else {
- sbd.append(obj);
- }
- }
-
- return sbd.toString();
- }
-
- /** format the column names and rows in table as markdown. */
- public static String format(String[] colNames, List<Row> data) {
- StringBuilder sbd = new StringBuilder();
- sbd.append(formatTitle(colNames));
-
- for (Row row : data) {
- sbd.append("\n").append(formatRows(row));
- }
-
- return sbd.toString();
- }
-
- /**
- * Convert column name array to SQL clause.
- *
- * <p>For example, columns "{a, b, c}" will be converted into a SQL-compatible select string
- * section: "`a`, `b`, `c`".
- *
- * @param colNames columns to convert
- * @return converted SQL clause.
- */
- public static String columnsToSqlClause(String[] colNames) {
- return Joiner.on("`,`").appendTo(new StringBuilder("`"), colNames).append("`").toString();
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/VectorTypes.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/VectorTypes.java
deleted file mode 100644
index ce49609..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/common/utils/VectorTypes.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.ml.common.linalg.DenseVector;
-import org.apache.flink.ml.common.linalg.SparseVector;
-import org.apache.flink.ml.common.linalg.Vector;
-
-/** Built-in vector types. */
-public class VectorTypes {
- /** <code>DenseVector</code> type information. */
- public static final TypeInformation<DenseVector> DENSE_VECTOR =
- TypeInformation.of(DenseVector.class);
-
- /** <code>SparseVector</code> type information. */
- public static final TypeInformation<SparseVector> SPARSE_VECTOR =
- TypeInformation.of(SparseVector.class);
-
- /**
- * <code>Vector</code> type information. For efficiency, use type information of sub-class
- * <code>DenseVector</code> and <code>SparseVector</code> as much as possible. When an operator
- * output both sub-class type of vectors, use this one.
- */
- public static final TypeInformation<Vector> VECTOR = TypeInformation.of(Vector.class);
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java
deleted file mode 100644
index 2efaca7..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/AlgoOperator.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.operator;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.api.misc.param.WithParams;
-import org.apache.flink.ml.params.shared.HasMLEnvironmentId;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.util.Preconditions;
-
-import java.io.Serializable;
-
-/**
- * Base class for algorithm operators.
- *
- * <p>Base class for the algorithm operators. It hosts the parameters and output tables of an
- * algorithm operator. Each AlgoOperator may have one or more output tables. One of the output table
- * is the primary output table which can be obtained by calling {@link #getOutput}. The other output
- * tables are side output tables that can be obtained by calling {@link #getSideOutputs()}.
- *
- * <p>The input of an AlgoOperator is defined in the subclasses of the AlgoOperator.
- *
- * @param <T> The class type of the {@link AlgoOperator} implementation itself
- */
-public abstract class AlgoOperator<T extends AlgoOperator<T>>
- implements WithParams<T>, HasMLEnvironmentId<T>, Serializable {
-
- /** Params for algorithms. */
- private Params params;
-
- /** The table held by operator. */
- private Table output = null;
-
- /** The side outputs of operator that be similar to the stream's side outputs. */
- private Table[] sideOutputs = null;
-
- /**
- * Construct the operator with empty Params.
- *
- * <p>This constructor is especially useful when users want to set parameters for the algorithm
- * operators. For example: SplitBatchOp is widely used in ML data pre-processing, which splits
- * one dataset into two dataset: training set and validation set. It is very convenient for us
- * to write code like this:
- *
- * <pre>{@code
- * new SplitBatchOp().setSplitRatio(0.9)
- * }</pre>
- */
- protected AlgoOperator() {
- this(null);
- }
-
- /** Construct the operator with the initial Params. */
- protected AlgoOperator(Params params) {
- if (null == params) {
- this.params = new Params();
- } else {
- this.params = params.clone();
- }
- }
-
- @Override
- public Params getParams() {
- return this.params;
- }
-
- /** Returns the table held by operator. */
- public Table getOutput() {
- return this.output;
- }
-
- /** Returns the side outputs. */
- public Table[] getSideOutputs() {
- return this.sideOutputs;
- }
-
- /**
- * Set the side outputs.
- *
- * @param sideOutputs the side outputs set the operator.
- */
- protected void setSideOutputs(Table[] sideOutputs) {
- this.sideOutputs = sideOutputs;
- }
-
- /**
- * Set the table held by operator.
- *
- * @param output the output table.
- */
- protected void setOutput(Table output) {
- this.output = output;
- }
-
- /** Returns the column names of the output table. */
- public String[] getColNames() {
- return getSchema().getFieldNames();
- }
-
- /** Returns the column types of the output table. */
- public TypeInformation<?>[] getColTypes() {
- return getSchema().getFieldTypes();
- }
-
- /**
- * Get the column names of the specified side-output table.
- *
- * @param index the index of the table.
- * @return the column types of the table.
- */
- public String[] getSideOutputColNames(int index) {
- checkSideOutputAccessibility(index);
-
- return sideOutputs[index].getSchema().getFieldNames();
- }
-
- /**
- * Get the column types of the specified side-output table.
- *
- * @param index the index of the table.
- * @return the column types of the table.
- */
- public TypeInformation<?>[] getSideOutputColTypes(int index) {
- checkSideOutputAccessibility(index);
-
- return sideOutputs[index].getSchema().getFieldTypes();
- }
-
- /** Returns the schema of the output table. */
- public TableSchema getSchema() {
- return this.getOutput().getSchema();
- }
-
- @Override
- public String toString() {
- return getOutput().toString();
- }
-
- protected static void checkOpSize(int size, AlgoOperator<?>... inputs) {
- Preconditions.checkNotNull(inputs, "Operators should not be null.");
- Preconditions.checkState(
- inputs.length == size,
- "The size of operators should be equal to " + size + ", current: " + inputs.length);
- }
-
- protected static void checkMinOpSize(int size, AlgoOperator<?>... inputs) {
- Preconditions.checkNotNull(inputs, "Operators should not be null.");
- Preconditions.checkState(
- inputs.length >= size,
- "The size of operators should be equal or greater than "
- + size
- + ", current: "
- + inputs.length);
- }
-
- private void checkSideOutputAccessibility(int index) {
- Preconditions.checkNotNull(sideOutputs, "There is not side-outputs in this AlgoOperator.");
- Preconditions.checkState(
- index >= 0 && index < sideOutputs.length,
- String.format("The index(%s) of side-outputs is out of bound.", index));
- Preconditions.checkNotNull(
- sideOutputs[index],
- String.format(
- "The %snd of side-outputs is null. Maybe the operator has not been linked.",
- index));
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java
deleted file mode 100644
index 1ddfdb3..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/BatchOperator.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.operator.batch;
-
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.operator.AlgoOperator;
-import org.apache.flink.ml.operator.batch.source.TableSourceBatchOp;
-import org.apache.flink.table.api.Table;
-
-/**
- * Base class of batch algorithm operators.
- *
- * <p>This class extends {@link AlgoOperator} to support data transmission between BatchOperators.
- */
-public abstract class BatchOperator<T extends BatchOperator<T>> extends AlgoOperator<T> {
-
- public BatchOperator() {
- super();
- }
-
- /**
- * The constructor of BatchOperator with {@link Params}.
- *
- * @param params the initial Params.
- */
- public BatchOperator(Params params) {
- super(params);
- }
-
- /**
- * Link to another {@link BatchOperator}.
- *
- * <p>Link the <code>next</code> BatchOperator using this BatchOperator as its input.
- *
- * <p>For example:
- *
- * <pre>{@code
- * BatchOperator a = ...;
- * BatchOperator b = ...;
- * BatchOperator c = a.link(b)
- * }</pre>
- *
- * <p>The BatchOperator <code>c</code> in the above code is the same instance as <code>b</code>
- * which takes <code>a</code> as its input. Note that BatchOperator <code>b</code> will be
- * changed to link from BatchOperator <code>a</code>.
- *
- * @param next The operator that will be modified to add this operator to its input.
- * @param <B> type of BatchOperator returned
- * @return the linked next
- * @see #linkFrom(BatchOperator[])
- */
- public <B extends BatchOperator<?>> B link(B next) {
- next.linkFrom(this);
- return next;
- }
-
- /**
- * Link from others {@link BatchOperator}.
- *
- * <p>Link this object to BatchOperator using the BatchOperators as its input.
- *
- * <p>For example:
- *
- * <pre>{@code
- * BatchOperator a = ...;
- * BatchOperator b = ...;
- * BatchOperator c = ...;
- *
- * BatchOperator d = c.linkFrom(a, b)
- * }</pre>
- *
- * <p>The <code>d</code> in the above code is the same instance as BatchOperator <code>c</code>
- * which takes both <code>a</code> and <code>b</code> as its input.
- *
- * <p>note: It is not recommended to linkFrom itself or linkFrom the same group inputs twice.
- *
- * @param inputs the linked inputs
- * @return the linked this object
- */
- public abstract T linkFrom(BatchOperator<?>... inputs);
-
- /**
- * create a new BatchOperator from table.
- *
- * @param table the input table
- * @return the new BatchOperator
- */
- public static BatchOperator<?> fromTable(Table table) {
- return new TableSourceBatchOp(table);
- }
-
- protected static BatchOperator<?> checkAndGetFirst(BatchOperator<?>... inputs) {
- checkOpSize(1, inputs);
- return inputs[0];
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java
deleted file mode 100644
index f53ae12..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/batch/source/TableSourceBatchOp.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.operator.batch.source;
-
-import org.apache.flink.ml.operator.batch.BatchOperator;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.util.Preconditions;
-
-/** Transform the Table to SourceBatchOp. */
-public final class TableSourceBatchOp extends BatchOperator<TableSourceBatchOp> {
-
- public TableSourceBatchOp(Table table) {
- super(null);
- Preconditions.checkArgument(table != null, "The source table cannot be null.");
- this.setOutput(table);
- }
-
- @Override
- public TableSourceBatchOp linkFrom(BatchOperator<?>... inputs) {
- throw new UnsupportedOperationException(
- "Table source operator should not have any upstream to link from.");
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java
deleted file mode 100644
index 373556c..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/StreamOperator.java
+++ /dev/null
@@ -1,114 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.operator.stream;
-
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.operator.AlgoOperator;
-import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp;
-import org.apache.flink.table.api.Table;
-
-/**
- * Base class of stream algorithm operators.
- *
- * <p>This class extends {@link AlgoOperator} to support data transmission between StreamOperator.
- */
-public abstract class StreamOperator<T extends StreamOperator<T>> extends AlgoOperator<T> {
-
- public StreamOperator() {
- super();
- }
-
- /**
- * The constructor of StreamOperator with {@link Params}.
- *
- * @param params the initial Params.
- */
- public StreamOperator(Params params) {
- super(params);
- }
-
- /**
- * Link to another {@link StreamOperator}.
- *
- * <p>Link the <code>next</code> StreamOperator using this StreamOperator as its input.
- *
- * <p>For example:
- *
- * <pre>{@code
- * StreamOperator a = ...;
- * StreamOperator b = ...;
- *
- * StreamOperator c = a.link(b)
- * }</pre>
- *
- * <p>The StreamOperator <code>c</code> in the above code is the same instance as <code>b</code>
- * which takes <code>a</code> as its input. Note that StreamOperator <code>b</code> will be
- * changed to link from StreamOperator <code>a</code>.
- *
- * @param next the linked StreamOperator
- * @param <S> type of StreamOperator returned
- * @return the linked next
- * @see #linkFrom(StreamOperator[])
- */
- public <S extends StreamOperator<?>> S link(S next) {
- next.linkFrom(this);
- return next;
- }
-
- /**
- * Link from others {@link StreamOperator}.
- *
- * <p>Link this object to StreamOperator using the StreamOperators as its input.
- *
- * <p>For example:
- *
- * <pre>{@code
- * StreamOperator a = ...;
- * StreamOperator b = ...;
- * StreamOperator c = ...;
- *
- * StreamOperator d = c.linkFrom(a, b)
- * }</pre>
- *
- * <p>The <code>d</code> in the above code is the same instance as StreamOperator <code>c</code>
- * which takes both <code>a</code> and <code>b</code> as its input.
- *
- * <p>note: It is not recommended to linkFrom itself or linkFrom the same group inputs twice.
- *
- * @param inputs the linked inputs
- * @return the linked this object
- */
- public abstract T linkFrom(StreamOperator<?>... inputs);
-
- /**
- * create a new StreamOperator from table.
- *
- * @param table the input table
- * @return the new StreamOperator
- */
- public static StreamOperator<?> fromTable(Table table) {
- return new TableSourceStreamOp(table);
- }
-
- protected static StreamOperator<?> checkAndGetFirst(StreamOperator<?>... inputs) {
- checkOpSize(1, inputs);
- return inputs[0];
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java
deleted file mode 100644
index 90a284e..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/operator/stream/source/TableSourceStreamOp.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.operator.stream.source;
-
-import org.apache.flink.ml.operator.stream.StreamOperator;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.util.Preconditions;
-
-/** Transform the Table to SourceStreamOp. */
-public final class TableSourceStreamOp extends StreamOperator<TableSourceStreamOp> {
-
- public TableSourceStreamOp(Table table) {
- super(null);
- Preconditions.checkArgument(table != null, "The source table cannot be null.");
- this.setOutput(table);
- }
-
- @Override
- public TableSourceStreamOp linkFrom(StreamOperator<?>... inputs) {
- throw new UnsupportedOperationException(
- "Table source operator should not have any upstream to link from.");
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java
deleted file mode 100644
index 4ea1b6c..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/HasMLEnvironmentId.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-import org.apache.flink.ml.common.MLEnvironmentFactory;
-
-/** An interface for classes with a parameter specifying the id of MLEnvironment. */
-public interface HasMLEnvironmentId<T> extends WithParams<T> {
-
- ParamInfo<Long> ML_ENVIRONMENT_ID =
- ParamInfoFactory.createParamInfo("MLEnvironmentId", Long.class)
- .setDescription("ID of ML environment.")
- .setHasDefaultValue(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID)
- .build();
-
- default Long getMLEnvironmentId() {
- return get(ML_ENVIRONMENT_ID);
- }
-
- default T setMLEnvironmentId(Long value) {
- return set(ML_ENVIRONMENT_ID, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCol.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCol.java
deleted file mode 100644
index e731b8e..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCol.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the name of the output column.
- *
- * @see HasOutputCols
- * @see HasOutputColDefaultAsNull
- * @see HasOutputColsDefaultAsNull
- */
-public interface HasOutputCol<T> extends WithParams<T> {
-
- ParamInfo<String> OUTPUT_COL =
- ParamInfoFactory.createParamInfo("outputCol", String.class)
- .setDescription("Name of the output column")
- .setRequired()
- .build();
-
- default String getOutputCol() {
- return get(OUTPUT_COL);
- }
-
- default T setOutputCol(String value) {
- return set(OUTPUT_COL, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColDefaultAsNull.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColDefaultAsNull.java
deleted file mode 100644
index 1846f89..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColDefaultAsNull.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying name of the output column with a null
- * default value.
- *
- * @see HasOutputCol
- * @see HasOutputCols
- * @see HasOutputColsDefaultAsNull
- */
-public interface HasOutputColDefaultAsNull<T> extends WithParams<T> {
-
- ParamInfo<String> OUTPUT_COL =
- ParamInfoFactory.createParamInfo("outputCol", String.class)
- .setDescription("Name of the output column")
- .setHasDefaultValue(null)
- .build();
-
- default String getOutputCol() {
- return get(OUTPUT_COL);
- }
-
- default T setOutputCol(String value) {
- return set(OUTPUT_COL, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCols.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCols.java
deleted file mode 100644
index 7bccc73..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputCols.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying names of multiple output columns.
- *
- * @see HasOutputCol
- * @see HasOutputColDefaultAsNull
- * @see HasOutputColsDefaultAsNull
- */
-public interface HasOutputCols<T> extends WithParams<T> {
-
- ParamInfo<String[]> OUTPUT_COLS =
- ParamInfoFactory.createParamInfo("outputCols", String[].class)
- .setDescription("Names of the output columns")
- .setRequired()
- .build();
-
- default String[] getOutputCols() {
- return get(OUTPUT_COLS);
- }
-
- default T setOutputCols(String... value) {
- return set(OUTPUT_COLS, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColsDefaultAsNull.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColsDefaultAsNull.java
deleted file mode 100644
index 2b9eacd..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasOutputColsDefaultAsNull.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying names of multiple output columns. The
- * default parameter value is null.
- *
- * @see HasOutputCol
- * @see HasOutputColDefaultAsNull
- * @see HasOutputCols
- */
-public interface HasOutputColsDefaultAsNull<T> extends WithParams<T> {
-
- ParamInfo<String[]> OUTPUT_COLS =
- ParamInfoFactory.createParamInfo("outputCols", String[].class)
- .setDescription("Names of the output columns")
- .setHasDefaultValue(null)
- .build();
-
- default String[] getOutputCols() {
- return get(OUTPUT_COLS);
- }
-
- default T setOutputCols(String... value) {
- return set(OUTPUT_COLS, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionCol.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionCol.java
deleted file mode 100644
index 1bfd493..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionCol.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/** An interface for classes with a parameter specifying the column name of the prediction. */
-public interface HasPredictionCol<T> extends WithParams<T> {
-
- ParamInfo<String> PREDICTION_COL =
- ParamInfoFactory.createParamInfo("predictionCol", String.class)
- .setDescription("Column name of prediction.")
- .setRequired()
- .build();
-
- default String getPredictionCol() {
- return get(PREDICTION_COL);
- }
-
- default T setPredictionCol(String value) {
- return set(PREDICTION_COL, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionDetailCol.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionDetailCol.java
deleted file mode 100644
index 4e409fa..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasPredictionDetailCol.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the column name of prediction detail.
- *
- * <p>The detail is the information of prediction result, such as the probability of each label in
- * classifier.
- */
-public interface HasPredictionDetailCol<T> extends WithParams<T> {
-
- ParamInfo<String> PREDICTION_DETAIL_COL =
- ParamInfoFactory.createParamInfo("predictionDetailCol", String.class)
- .setDescription(
- "Column name of prediction result, it will include detailed info.")
- .build();
-
- default String getPredictionDetailCol() {
- return get(PREDICTION_DETAIL_COL);
- }
-
- default T setPredictionDetailCol(String value) {
- return set(PREDICTION_DETAIL_COL, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasReservedCols.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasReservedCols.java
deleted file mode 100644
index f06689e..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasReservedCols.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the names of the columns to be retained in
- * the output table.
- */
-public interface HasReservedCols<T> extends WithParams<T> {
-
- ParamInfo<String[]> RESERVED_COLS =
- ParamInfoFactory.createParamInfo("reservedCols", String[].class)
- .setDescription("Names of the columns to be retained in the output table")
- .setHasDefaultValue(null)
- .build();
-
- default String[] getReservedCols() {
- return get(RESERVED_COLS);
- }
-
- default T setReservedCols(String... value) {
- return set(RESERVED_COLS, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCol.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCol.java
deleted file mode 100644
index 88560dc..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCol.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the name of the table column.
- *
- * @see HasSelectedColDefaultAsNull
- * @see HasSelectedCols
- * @see HasSelectedColsDefaultAsNull
- */
-public interface HasSelectedCol<T> extends WithParams<T> {
-
- ParamInfo<String> SELECTED_COL =
- ParamInfoFactory.createParamInfo("selectedCol", String.class)
- .setDescription("Name of the selected column used for processing")
- .setRequired()
- .build();
-
- default String getSelectedCol() {
- return get(SELECTED_COL);
- }
-
- default T setSelectedCol(String value) {
- return set(SELECTED_COL, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColDefaultAsNull.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColDefaultAsNull.java
deleted file mode 100644
index 72a1dd7..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColDefaultAsNull.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the name of the table column with null
- * default value.
- *
- * @see HasSelectedCol
- * @see HasSelectedCols
- * @see HasSelectedColsDefaultAsNull
- */
-public interface HasSelectedColDefaultAsNull<T> extends WithParams<T> {
-
- ParamInfo<String> SELECTED_COL =
- ParamInfoFactory.createParamInfo("selectedCol", String.class)
- .setDescription("Name of the selected column used for processing")
- .setHasDefaultValue(null)
- .build();
-
- default String getSelectedCol() {
- return get(SELECTED_COL);
- }
-
- default T setSelectedCol(String value) {
- return set(SELECTED_COL, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCols.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCols.java
deleted file mode 100644
index 68b5bba..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedCols.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the name of multiple table columns.
- *
- * @see HasSelectedCol
- * @see HasSelectedColDefaultAsNull
- * @see HasSelectedColsDefaultAsNull
- */
-public interface HasSelectedCols<T> extends WithParams<T> {
-
- ParamInfo<String[]> SELECTED_COLS =
- ParamInfoFactory.createParamInfo("selectedCols", String[].class)
- .setDescription("Names of the columns used for processing")
- .setRequired()
- .build();
-
- default String[] getSelectedCols() {
- return get(SELECTED_COLS);
- }
-
- default T setSelectedCols(String... value) {
- return set(SELECTED_COLS, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColsDefaultAsNull.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColsDefaultAsNull.java
deleted file mode 100644
index 5e2801f..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/params/shared/colname/HasSelectedColsDefaultAsNull.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.params.shared.colname;
-
-import org.apache.flink.ml.api.misc.param.ParamInfo;
-import org.apache.flink.ml.api.misc.param.ParamInfoFactory;
-import org.apache.flink.ml.api.misc.param.WithParams;
-
-/**
- * An interface for classes with a parameter specifying the name of multiple table columns with null
- * default value.
- *
- * @see HasSelectedCol
- * @see HasSelectedColDefaultAsNull
- * @see HasSelectedCols
- */
-public interface HasSelectedColsDefaultAsNull<T> extends WithParams<T> {
-
- ParamInfo<String[]> SELECTED_COLS =
- ParamInfoFactory.createParamInfo("selectedCols", String[].class)
- .setDescription("Names of the columns used for processing")
- .setHasDefaultValue(null)
- .build();
-
- default String[] getSelectedCols() {
- return get(SELECTED_COLS);
- }
-
- default T setSelectedCols(String... value) {
- return set(SELECTED_COLS, value);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java
deleted file mode 100644
index cf7d302..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/EstimatorBase.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.pipeline;
-
-import org.apache.flink.ml.api.core.Estimator;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.operator.batch.BatchOperator;
-import org.apache.flink.ml.operator.batch.source.TableSourceBatchOp;
-import org.apache.flink.ml.operator.stream.StreamOperator;
-import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.table.api.internal.TableImpl;
-import org.apache.flink.util.Preconditions;
-
-/**
- * The base class for estimator implementations.
- *
- * @param <E> A subclass of the {@link EstimatorBase}, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams}
- * @param <M> class type of the {@link ModelBase} this Estimator produces.
- */
-public abstract class EstimatorBase<E extends EstimatorBase<E, M>, M extends ModelBase<M>>
- extends PipelineStageBase<E> implements Estimator<E, M> {
-
- public EstimatorBase() {
- super();
- }
-
- public EstimatorBase(Params params) {
- super(params);
- }
-
- @Override
- public M fit(TableEnvironment tEnv, Table input) {
- Preconditions.checkArgument(input != null, "Input CAN NOT BE null!");
- Preconditions.checkArgument(
- tableEnvOf(input) == tEnv,
- "The input table is not in the specified table environment.");
- return fit(input);
- }
-
- /**
- * Train and produce a {@link ModelBase} which fits the records in the given {@link Table}.
- *
- * @param input the table with records to train the Model.
- * @return a model trained to fit on the given Table.
- */
- public M fit(Table input) {
- Preconditions.checkArgument(input != null, "Input CAN NOT BE null!");
- if (((TableImpl) input).getTableEnvironment() instanceof StreamTableEnvironment) {
- TableSourceStreamOp source = new TableSourceStreamOp(input);
- if (this.params.contains(ML_ENVIRONMENT_ID)) {
- source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID));
- }
- return fit(source);
- } else {
- TableSourceBatchOp source = new TableSourceBatchOp(input);
- if (this.params.contains(ML_ENVIRONMENT_ID)) {
- source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID));
- }
- return fit(source);
- }
- }
-
- /**
- * Train and produce a {@link ModelBase} which fits the records from the given {@link
- * BatchOperator}.
- *
- * @param input the table with records to train the Model.
- * @return a model trained to fit on the given Table.
- */
- protected abstract M fit(BatchOperator input);
-
- /**
- * Online learning and produce {@link ModelBase} series which fit the streaming records from the
- * given {@link StreamOperator}.
- *
- * @param input the StreamOperator with streaming records to online train the Model series.
- * @return the model series trained to fit on the streaming data from given StreamOperator.
- */
- protected M fit(StreamOperator input) {
- throw new UnsupportedOperationException("NOT supported yet!");
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java
deleted file mode 100644
index f873c3b..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/ModelBase.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.pipeline;
-
-import org.apache.flink.ml.api.core.Model;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.table.api.Table;
-
-/**
- * The base class for a machine learning model.
- *
- * @param <M> The class type of the {@link ModelBase} implementation itself
- */
-public abstract class ModelBase<M extends ModelBase<M>> extends TransformerBase<M>
- implements Model<M> {
-
- protected Table modelData;
-
- public ModelBase() {
- super();
- }
-
- public ModelBase(Params params) {
- super(params);
- }
-
- /**
- * Get model data as Table representation.
- *
- * @return the Table
- */
- public Table getModelData() {
- return this.modelData;
- }
-
- /**
- * Set the model data using the Table.
- *
- * @param modelData the Table.
- * @return {@link ModelBase} itself
- */
- public M setModelData(Table modelData) {
- this.modelData = modelData;
- return (M) this;
- }
-
- @Override
- public M clone() throws CloneNotSupportedException {
- return (M) super.clone().setModelData(this.modelData);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java
deleted file mode 100644
index e6a957a..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/PipelineStageBase.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.pipeline;
-
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.api.misc.param.WithParams;
-import org.apache.flink.ml.params.shared.HasMLEnvironmentId;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-import org.apache.flink.table.api.internal.TableImpl;
-
-/**
- * The base class for a stage in a pipeline, either an [[EstimatorBase]] or a [[TransformerBase]].
- *
- * <p>The PipelineStageBase maintains the parameters for the stage. A default constructor is needed
- * in order to restore a pipeline stage.
- *
- * @param <S> The class type of the {@link PipelineStageBase} implementation itself, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams} and Cloneable.
- */
-public abstract class PipelineStageBase<S extends PipelineStageBase<S>>
- implements WithParams<S>, HasMLEnvironmentId<S>, Cloneable {
- protected Params params;
-
- public PipelineStageBase() {
- this(null);
- }
-
- public PipelineStageBase(Params params) {
- if (null == params) {
- this.params = new Params();
- } else {
- this.params = params.clone();
- }
- }
-
- @Override
- public Params getParams() {
- if (null == this.params) {
- this.params = new Params();
- }
- return this.params;
- }
-
- @Override
- public S clone() throws CloneNotSupportedException {
- PipelineStageBase result = (PipelineStageBase) super.clone();
- result.params = this.params.clone();
- return (S) result;
- }
-
- protected static TableEnvironment tableEnvOf(Table table) {
- return ((TableImpl) table).getTableEnvironment();
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java b/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java
deleted file mode 100644
index a1e546f..0000000
--- a/flink-ml-parent/flink-ml-lib/src/main/java/org/apache/flink/ml/pipeline/TransformerBase.java
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.pipeline;
-
-import org.apache.flink.ml.api.core.Transformer;
-import org.apache.flink.ml.api.misc.param.Params;
-import org.apache.flink.ml.operator.batch.BatchOperator;
-import org.apache.flink.ml.operator.batch.source.TableSourceBatchOp;
-import org.apache.flink.ml.operator.stream.StreamOperator;
-import org.apache.flink.ml.operator.stream.source.TableSourceStreamOp;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableEnvironment;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-import org.apache.flink.util.Preconditions;
-
-/**
- * The base class for transformer implementations.
- *
- * @param <T> A subclass of {@link TransformerBase}, used by {@link
- * org.apache.flink.ml.api.misc.param.WithParams}
- */
-public abstract class TransformerBase<T extends TransformerBase<T>> extends PipelineStageBase<T>
- implements Transformer<T> {
-
- public TransformerBase() {
- super();
- }
-
- public TransformerBase(Params params) {
- super(params);
- }
-
- @Override
- public Table transform(TableEnvironment tEnv, Table input) {
- Preconditions.checkArgument(input != null, "Input CAN NOT BE null!");
- Preconditions.checkArgument(
- tableEnvOf(input) == tEnv,
- "The input table is not in the specified table environment.");
- return transform(input);
- }
-
- /**
- * Applies the transformer on the input table, and returns the result table.
- *
- * @param input the table to be transformed
- * @return the transformed table
- */
- public Table transform(Table input) {
- Preconditions.checkArgument(input != null, "Input CAN NOT BE null!");
- if (tableEnvOf(input) instanceof StreamTableEnvironment) {
- TableSourceStreamOp source = new TableSourceStreamOp(input);
- if (this.params.contains(ML_ENVIRONMENT_ID)) {
- source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID));
- }
- return transform(source).getOutput();
- } else {
- TableSourceBatchOp source = new TableSourceBatchOp(input);
- if (this.params.contains(ML_ENVIRONMENT_ID)) {
- source.setMLEnvironmentId(this.params.get(ML_ENVIRONMENT_ID));
- }
- return transform(source).getOutput();
- }
- }
-
- /**
- * Applies the transformer on the input batch data from BatchOperator, and returns the batch
- * result data with BatchOperator.
- *
- * @param input the input batch data from BatchOperator
- * @return the transformed batch result data
- */
- protected abstract BatchOperator transform(BatchOperator input);
-
- /**
- * Applies the transformer on the input streaming data from StreamOperator, and returns the
- * streaming result data with StreamOperator.
- *
- * @param input the input streaming data from StreamOperator
- * @return the transformed streaming result data
- */
- protected abstract StreamOperator transform(StreamOperator input);
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java
deleted file mode 100644
index bbc1c34..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/MLEnvironmentTest.java
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common;
-
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.EnvironmentSettings;
-import org.apache.flink.table.api.bridge.java.BatchTableEnvironment;
-import org.apache.flink.table.api.bridge.java.StreamTableEnvironment;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-
-/** Test cases for MLEnvironment. */
-public class MLEnvironmentTest {
- @Test
- public void testDefaultConstructor() {
- MLEnvironment mlEnvironment = new MLEnvironment();
- Assert.assertNotNull(mlEnvironment.getExecutionEnvironment());
- Assert.assertNotNull(mlEnvironment.getBatchTableEnvironment());
- Assert.assertNotNull(mlEnvironment.getStreamExecutionEnvironment());
- Assert.assertNotNull(mlEnvironment.getStreamTableEnvironment());
- }
-
- @Test
- public void testConstructWithBatchEnv() {
- ExecutionEnvironment executionEnvironment = ExecutionEnvironment.getExecutionEnvironment();
- BatchTableEnvironment batchTableEnvironment =
- BatchTableEnvironment.create(executionEnvironment);
-
- MLEnvironment mlEnvironment =
- new MLEnvironment(executionEnvironment, batchTableEnvironment);
-
- Assert.assertSame(mlEnvironment.getExecutionEnvironment(), executionEnvironment);
- Assert.assertSame(mlEnvironment.getBatchTableEnvironment(), batchTableEnvironment);
- }
-
- @Test
- public void testConstructWithStreamEnv() {
- StreamExecutionEnvironment streamExecutionEnvironment =
- StreamExecutionEnvironment.getExecutionEnvironment();
- StreamTableEnvironment streamTableEnvironment =
- StreamTableEnvironment.create(
- streamExecutionEnvironment,
- EnvironmentSettings.newInstance().useOldPlanner().build());
-
- MLEnvironment mlEnvironment =
- new MLEnvironment(streamExecutionEnvironment, streamTableEnvironment);
-
- Assert.assertSame(
- mlEnvironment.getStreamExecutionEnvironment(), streamExecutionEnvironment);
- Assert.assertSame(mlEnvironment.getStreamTableEnvironment(), streamTableEnvironment);
- }
-
- @Test
- public void testRemoveDefaultMLEnvironment() {
- MLEnvironment defaultEnv = MLEnvironmentFactory.getDefault();
- MLEnvironmentFactory.remove(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID);
- assertEquals(
- "The default MLEnvironment should not have been removed",
- defaultEnv,
- MLEnvironmentFactory.get(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID));
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/BLASTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/BLASTest.java
deleted file mode 100644
index 13e67ff..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/BLASTest.java
+++ /dev/null
@@ -1,186 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-/** The test cases for {@link BLAS}. */
-public class BLASTest {
- private static final double TOL = 1.0e-8;
- private DenseMatrix mat = new DenseMatrix(2, 3, new double[] {1, 4, 2, 5, 3, 6});
- private DenseVector dv1 = new DenseVector(new double[] {1, 2});
- private DenseVector dv2 = new DenseVector(new double[] {1, 2, 3});
- private SparseVector spv1 = new SparseVector(2, new int[] {0, 1}, new double[] {1, 2});
- private SparseVector spv2 = new SparseVector(3, new int[] {0, 2}, new double[] {1, 3});
-
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testAsum() throws Exception {
- Assert.assertEquals(BLAS.asum(dv1), 3.0, TOL);
- Assert.assertEquals(BLAS.asum(spv1), 3.0, TOL);
- }
-
- @Test
- public void testScal() throws Exception {
- DenseVector v1 = dv1.clone();
- BLAS.scal(0.5, v1);
- Assert.assertArrayEquals(v1.getData(), new double[] {0.5, 1.0}, TOL);
-
- SparseVector v2 = spv1.clone();
- BLAS.scal(0.5, v2);
- Assert.assertArrayEquals(v2.getIndices(), spv1.getIndices());
- Assert.assertArrayEquals(v2.getValues(), new double[] {0.5, 1.0}, TOL);
- }
-
- @Test
- public void testDot() throws Exception {
- DenseVector v = DenseVector.ones(2);
- Assert.assertEquals(BLAS.dot(dv1, v), 3.0, TOL);
- }
-
- @Test
- public void testAxpy() throws Exception {
- DenseVector v = DenseVector.ones(2);
- BLAS.axpy(1.0, dv1, v);
- Assert.assertArrayEquals(v.getData(), new double[] {2, 3}, TOL);
- BLAS.axpy(1.0, spv1, v);
- Assert.assertArrayEquals(v.getData(), new double[] {3, 5}, TOL);
- BLAS.axpy(1, 1.0, new double[] {1}, 0, v.getData(), 1);
- Assert.assertArrayEquals(v.getData(), new double[] {3, 6}, TOL);
- }
-
- private DenseMatrix simpleMM(DenseMatrix m1, DenseMatrix m2) {
- DenseMatrix mm = new DenseMatrix(m1.numRows(), m2.numCols());
- for (int i = 0; i < m1.numRows(); i++) {
- for (int j = 0; j < m2.numCols(); j++) {
- double s = 0.;
- for (int k = 0; k < m1.numCols(); k++) {
- s += m1.get(i, k) * m2.get(k, j);
- }
- mm.set(i, j, s);
- }
- }
- return mm;
- }
-
- @Test
- public void testGemm() throws Exception {
- DenseMatrix m32 = DenseMatrix.rand(3, 2);
- DenseMatrix m24 = DenseMatrix.rand(2, 4);
- DenseMatrix m34 = DenseMatrix.rand(3, 4);
- DenseMatrix m42 = DenseMatrix.rand(4, 2);
- DenseMatrix m43 = DenseMatrix.rand(4, 3);
-
- DenseMatrix a34 = DenseMatrix.zeros(3, 4);
- BLAS.gemm(1.0, m32, false, m24, false, 0., a34);
- Assert.assertArrayEquals(a34.getData(), simpleMM(m32, m24).getData(), TOL);
-
- BLAS.gemm(1.0, m32, false, m42, true, 0., a34);
- Assert.assertArrayEquals(a34.getData(), simpleMM(m32, m42.transpose()).getData(), TOL);
-
- DenseMatrix a24 = DenseMatrix.zeros(2, 4);
- BLAS.gemm(1.0, m32, true, m34, false, 0., a24);
- Assert.assertArrayEquals(a24.getData(), simpleMM(m32.transpose(), m34).getData(), TOL);
-
- BLAS.gemm(1.0, m32, true, m43, true, 0., a24);
- Assert.assertArrayEquals(
- a24.getData(), simpleMM(m32.transpose(), m43.transpose()).getData(), TOL);
- }
-
- @Test
- public void testGemmSizeCheck() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- DenseMatrix m32 = DenseMatrix.rand(3, 2);
- DenseMatrix m42 = DenseMatrix.rand(4, 2);
- DenseMatrix a34 = DenseMatrix.zeros(3, 4);
- BLAS.gemm(1.0, m32, false, m42, false, 0., a34);
- }
-
- @Test
- public void testGemmTransposeSizeCheck() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- DenseMatrix m32 = DenseMatrix.rand(3, 2);
- DenseMatrix m42 = DenseMatrix.rand(4, 2);
- DenseMatrix a34 = DenseMatrix.zeros(3, 4);
- BLAS.gemm(1.0, m32, true, m42, true, 0., a34);
- }
-
- @Test
- public void testGemvDense() throws Exception {
- DenseVector y1 = DenseVector.ones(2);
- BLAS.gemv(2.0, mat, false, dv2, 0., y1);
- Assert.assertArrayEquals(new double[] {28, 64}, y1.data, TOL);
-
- DenseVector y2 = DenseVector.ones(2);
- BLAS.gemv(2.0, mat, false, dv2, 1., y2);
- Assert.assertArrayEquals(new double[] {29, 65}, y2.data, TOL);
- }
-
- @Test
- public void testGemvDenseTranspose() throws Exception {
- DenseVector y1 = DenseVector.ones(3);
- BLAS.gemv(1.0, mat, true, dv1, 0., y1);
- Assert.assertArrayEquals(new double[] {9, 12, 15}, y1.data, TOL);
-
- DenseVector y2 = DenseVector.ones(3);
- BLAS.gemv(1.0, mat, true, dv1, 1., y2);
- Assert.assertArrayEquals(new double[] {10, 13, 16}, y2.data, TOL);
- }
-
- @Test
- public void testGemvSparse() throws Exception {
- DenseVector y1 = DenseVector.ones(2);
- BLAS.gemv(2.0, mat, false, spv2, 0., y1);
- Assert.assertArrayEquals(new double[] {20, 44}, y1.data, TOL);
-
- DenseVector y2 = DenseVector.ones(2);
- BLAS.gemv(2.0, mat, false, spv2, 1., y2);
- Assert.assertArrayEquals(new double[] {21, 45}, y2.data, TOL);
- }
-
- @Test
- public void testGemvSparseTranspose() throws Exception {
- DenseVector y1 = DenseVector.ones(3);
- BLAS.gemv(2.0, mat, true, spv1, 0., y1);
- Assert.assertArrayEquals(new double[] {18, 24, 30}, y1.data, TOL);
-
- DenseVector y2 = DenseVector.ones(3);
- BLAS.gemv(2.0, mat, true, spv1, 1., y2);
- Assert.assertArrayEquals(new double[] {19, 25, 31}, y2.data, TOL);
- }
-
- @Test
- public void testGemvSizeCheck() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- DenseVector y = DenseVector.ones(2);
- BLAS.gemv(2.0, mat, false, dv1, 0., y);
- }
-
- @Test
- public void testGemvTransposeSizeCheck() throws Exception {
- thrown.expect(IllegalArgumentException.class);
- DenseVector y = DenseVector.ones(2);
- BLAS.gemv(2.0, mat, true, dv1, 0., y);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseMatrixTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseMatrixTest.java
deleted file mode 100644
index 40cb9a4..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseMatrixTest.java
+++ /dev/null
@@ -1,195 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/** Test cases for DenseMatrix. */
-public class DenseMatrixTest {
-
- private static final double TOL = 1.0e-6;
-
- private static void assertEqual2D(double[][] matA, double[][] matB) {
- assert (matA.length == matB.length);
- assert (matA[0].length == matB[0].length);
- int m = matA.length;
- int n = matA[0].length;
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- Assert.assertEquals(matA[i][j], matB[i][j], TOL);
- }
- }
- }
-
- private static double[][] simpleMM(double[][] matA, double[][] matB) {
- int m = matA.length;
- int n = matB[0].length;
- int k = matA[0].length;
- double[][] matC = new double[m][n];
- for (int i = 0; i < m; i++) {
- for (int j = 0; j < n; j++) {
- matC[i][j] = 0.;
- for (int l = 0; l < k; l++) {
- matC[i][j] += matA[i][l] * matB[l][j];
- }
- }
- }
- return matC;
- }
-
- private static double[] simpleMV(double[][] matA, double[] x) {
- int m = matA.length;
- int n = matA[0].length;
- assert (n == x.length);
- double[] y = new double[m];
- for (int i = 0; i < m; i++) {
- y[i] = 0.;
- for (int j = 0; j < n; j++) {
- y[i] += matA[i][j] * x[j];
- }
- }
- return y;
- }
-
- @Test
- public void testPlusEquals() throws Exception {
- DenseMatrix matA =
- new DenseMatrix(
- new double[][] {
- new double[] {1, 3, 5},
- new double[] {2, 4, 6},
- });
- DenseMatrix matB = DenseMatrix.ones(2, 3);
- matA.plusEquals(matB);
- Assert.assertArrayEquals(matA.getData(), new double[] {2, 3, 4, 5, 6, 7}, TOL);
- matA.plusEquals(1.0);
- Assert.assertArrayEquals(matA.getData(), new double[] {3, 4, 5, 6, 7, 8}, TOL);
- }
-
- @Test
- public void testMinusEquals() throws Exception {
- DenseMatrix matA =
- new DenseMatrix(
- new double[][] {
- new double[] {1, 3, 5},
- new double[] {2, 4, 6},
- });
- DenseMatrix matB = DenseMatrix.ones(2, 3);
- matA.minusEquals(matB);
- Assert.assertArrayEquals(matA.getData(), new double[] {0, 1, 2, 3, 4, 5}, TOL);
- }
-
- @Test
- public void testPlus() throws Exception {
- DenseMatrix matA =
- new DenseMatrix(
- new double[][] {
- new double[] {1, 3, 5},
- new double[] {2, 4, 6},
- });
- DenseMatrix matB = DenseMatrix.ones(2, 3);
- DenseMatrix matC = matA.plus(matB);
- Assert.assertArrayEquals(matC.getData(), new double[] {2, 3, 4, 5, 6, 7}, TOL);
- DenseMatrix matD = matA.plus(1.0);
- Assert.assertArrayEquals(matD.getData(), new double[] {2, 3, 4, 5, 6, 7}, TOL);
- }
-
- @Test
- public void testMinus() throws Exception {
- DenseMatrix matA =
- new DenseMatrix(
- new double[][] {
- new double[] {1, 3, 5},
- new double[] {2, 4, 6},
- });
- DenseMatrix matB = DenseMatrix.ones(2, 3);
- DenseMatrix matC = matA.minus(matB);
- Assert.assertArrayEquals(matC.getData(), new double[] {0, 1, 2, 3, 4, 5}, TOL);
- }
-
- @Test
- public void testMM() throws Exception {
- DenseMatrix matA = DenseMatrix.rand(4, 3);
- DenseMatrix matB = DenseMatrix.rand(3, 5);
- DenseMatrix matC = matA.multiplies(matB);
- assertEqual2D(
- matC.getArrayCopy2D(), simpleMM(matA.getArrayCopy2D(), matB.getArrayCopy2D()));
-
- DenseMatrix matD = new DenseMatrix(5, 4);
- BLAS.gemm(1., matB, true, matA, true, 0., matD);
- Assert.assertArrayEquals(matD.transpose().getData(), matC.data, TOL);
- }
-
- @Test
- public void testMV() throws Exception {
- DenseMatrix matA = DenseMatrix.rand(4, 3);
- DenseVector x = DenseVector.ones(3);
- DenseVector y = matA.multiplies(x);
- Assert.assertArrayEquals(y.getData(), simpleMV(matA.getArrayCopy2D(), x.getData()), TOL);
-
- SparseVector x2 = new SparseVector(3, new int[] {0, 1, 2}, new double[] {1, 1, 1});
- DenseVector y2 = matA.multiplies(x2);
- Assert.assertArrayEquals(y2.getData(), y.getData(), TOL);
- }
-
- @Test
- public void testDataSelection() throws Exception {
- DenseMatrix mat =
- new DenseMatrix(
- new double[][] {
- new double[] {1, 2, 3},
- new double[] {4, 5, 6},
- new double[] {7, 8, 9},
- });
- DenseMatrix sub1 = mat.selectRows(new int[] {1});
- DenseMatrix sub2 = mat.getSubMatrix(1, 2, 1, 2);
- Assert.assertEquals(sub1.numRows(), 1);
- Assert.assertEquals(sub1.numCols(), 3);
- Assert.assertEquals(sub2.numRows(), 1);
- Assert.assertEquals(sub2.numCols(), 1);
- Assert.assertArrayEquals(sub1.getData(), new double[] {4, 5, 6}, TOL);
- Assert.assertArrayEquals(sub2.getData(), new double[] {5}, TOL);
-
- double[] row = mat.getRow(1);
- double[] col = mat.getColumn(1);
- Assert.assertArrayEquals(row, new double[] {4, 5, 6}, 0.);
- Assert.assertArrayEquals(col, new double[] {2, 5, 8}, 0.);
- }
-
- @Test
- public void testSum() throws Exception {
- DenseMatrix matA = DenseMatrix.ones(3, 2);
- Assert.assertEquals(matA.sum(), 6.0, TOL);
- }
-
- @Test
- public void testRowMajorFormat() throws Exception {
- double[] data = new double[] {1, 2, 3, 4, 5, 6};
- DenseMatrix matA = new DenseMatrix(2, 3, data, true);
- Assert.assertArrayEquals(data, new double[] {1, 4, 2, 5, 3, 6}, 0.);
- Assert.assertArrayEquals(matA.getData(), new double[] {1, 4, 2, 5, 3, 6}, 0.);
-
- data = new double[] {1, 2, 3, 4};
- matA = new DenseMatrix(2, 2, data, true);
- Assert.assertArrayEquals(data, new double[] {1, 3, 2, 4}, 0.);
- Assert.assertArrayEquals(matA.getData(), new double[] {1, 3, 2, 4}, 0.);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseVectorTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseVectorTest.java
deleted file mode 100644
index 7859317..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/DenseVectorTest.java
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/** Test cases for DenseVector. */
-public class DenseVectorTest {
- private static final double TOL = 1.0e-6;
-
- @Test
- public void testSize() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- Assert.assertEquals(vec.size(), 3);
- }
-
- @Test
- public void testNormL1() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- Assert.assertEquals(vec.normL1(), 6, 0);
- }
-
- @Test
- public void testNormMax() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- Assert.assertEquals(vec.normInf(), 3, 0);
- }
-
- @Test
- public void testNormL2() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- Assert.assertEquals(vec.normL2(), Math.sqrt(1 + 4 + 9), TOL);
- }
-
- @Test
- public void testNormL2Square() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- Assert.assertEquals(vec.normL2Square(), 1 + 4 + 9, TOL);
- }
-
- @Test
- public void testSlice() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- DenseVector sliced = vec.slice(new int[] {0, 2});
- Assert.assertArrayEquals(new double[] {1, -3}, sliced.getData(), 0);
- }
-
- @Test
- public void testMinus() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- DenseVector d = new DenseVector(new double[] {1, 2, 1});
- DenseVector vec2 = vec.minus(d);
- Assert.assertArrayEquals(vec.getData(), new double[] {1, 2, -3}, 0);
- Assert.assertArrayEquals(vec2.getData(), new double[] {0, 0, -4}, TOL);
- vec.minusEqual(d);
- Assert.assertArrayEquals(vec.getData(), new double[] {0, 0, -4}, TOL);
- }
-
- @Test
- public void testPlus() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- DenseVector d = new DenseVector(new double[] {1, 2, 1});
- DenseVector vec2 = vec.plus(d);
- Assert.assertArrayEquals(vec.getData(), new double[] {1, 2, -3}, 0);
- Assert.assertArrayEquals(vec2.getData(), new double[] {2, 4, -2}, TOL);
- vec.plusEqual(d);
- Assert.assertArrayEquals(vec.getData(), new double[] {2, 4, -2}, TOL);
- }
-
- @Test
- public void testPlusScaleEqual() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- DenseVector vec2 = new DenseVector(new double[] {1, 0, 2});
- vec.plusScaleEqual(vec2, 2.);
- Assert.assertArrayEquals(vec.getData(), new double[] {3, 2, 1}, TOL);
- }
-
- @Test
- public void testDot() throws Exception {
- DenseVector vec1 = new DenseVector(new double[] {1, 2, -3});
- DenseVector vec2 = new DenseVector(new double[] {3, 2, 1});
- Assert.assertEquals(vec1.dot(vec2), 3 + 4 - 3, TOL);
- }
-
- @Test
- public void testPrefix() throws Exception {
- DenseVector vec1 = new DenseVector(new double[] {1, 2, -3});
- DenseVector vec2 = vec1.prefix(0);
- Assert.assertArrayEquals(vec2.getData(), new double[] {0, 1, 2, -3}, 0);
- }
-
- @Test
- public void testAppend() throws Exception {
- DenseVector vec1 = new DenseVector(new double[] {1, 2, -3});
- DenseVector vec2 = vec1.append(0);
- Assert.assertArrayEquals(vec2.getData(), new double[] {1, 2, -3, 0}, 0);
- }
-
- @Test
- public void testOuter() throws Exception {
- DenseVector vec1 = new DenseVector(new double[] {1, 2, -3});
- DenseVector vec2 = new DenseVector(new double[] {3, 2, 1});
- DenseMatrix outer = vec1.outer(vec2);
- Assert.assertArrayEquals(
- outer.getArrayCopy1D(true), new double[] {3, 2, 1, 6, 4, 2, -9, -6, -3}, TOL);
- }
-
- @Test
- public void testNormalize() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- vec.normalizeEqual(1.0);
- Assert.assertArrayEquals(vec.getData(), new double[] {1. / 6, 2. / 6, -3. / 6}, TOL);
- }
-
- @Test
- public void testStandardize() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- vec.standardizeEqual(1.0, 1.0);
- Assert.assertArrayEquals(vec.getData(), new double[] {0, 1, -4}, TOL);
- }
-
- @Test
- public void testIterator() throws Exception {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- VectorIterator iterator = vec.iterator();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 0);
- Assert.assertEquals(iterator.getValue(), 1, 0);
- iterator.next();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 1);
- Assert.assertEquals(iterator.getValue(), 2, 0);
- iterator.next();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 2);
- Assert.assertEquals(iterator.getValue(), -3, 0);
- iterator.next();
- Assert.assertFalse(iterator.hasNext());
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/MatVecOpTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/MatVecOpTest.java
deleted file mode 100644
index bbcd6d8..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/MatVecOpTest.java
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-/** Test cases for {@link MatVecOp}. */
-public class MatVecOpTest {
- private static final double TOL = 1.0e-6;
- private DenseVector dv;
- private SparseVector sv;
-
- @Before
- public void setUp() throws Exception {
- dv = new DenseVector(new double[] {1, 2, 3, 4});
- sv = new SparseVector(4, new int[] {0, 2}, new double[] {1., 1.});
- }
-
- @Test
- public void testPlus() throws Exception {
- Vector plusResult1 = MatVecOp.plus(dv, sv);
- Vector plusResult2 = MatVecOp.plus(sv, dv);
- Vector plusResult3 = MatVecOp.plus(sv, sv);
- Vector plusResult4 = MatVecOp.plus(dv, dv);
- Assert.assertTrue(plusResult1 instanceof DenseVector);
- Assert.assertTrue(plusResult2 instanceof DenseVector);
- Assert.assertTrue(plusResult3 instanceof SparseVector);
- Assert.assertTrue(plusResult4 instanceof DenseVector);
- Assert.assertArrayEquals(
- ((DenseVector) plusResult1).getData(), new double[] {2, 2, 4, 4}, TOL);
- Assert.assertArrayEquals(
- ((DenseVector) plusResult2).getData(), new double[] {2, 2, 4, 4}, TOL);
- Assert.assertArrayEquals(((SparseVector) plusResult3).getIndices(), new int[] {0, 2});
- Assert.assertArrayEquals(
- ((SparseVector) plusResult3).getValues(), new double[] {2., 2.}, TOL);
- Assert.assertArrayEquals(
- ((DenseVector) plusResult4).getData(), new double[] {2, 4, 6, 8}, TOL);
- }
-
- @Test
- public void testMinus() throws Exception {
- Vector minusResult1 = MatVecOp.minus(dv, sv);
- Vector minusResult2 = MatVecOp.minus(sv, dv);
- Vector minusResult3 = MatVecOp.minus(sv, sv);
- Vector minusResult4 = MatVecOp.minus(dv, dv);
- Assert.assertTrue(minusResult1 instanceof DenseVector);
- Assert.assertTrue(minusResult2 instanceof DenseVector);
- Assert.assertTrue(minusResult3 instanceof SparseVector);
- Assert.assertTrue(minusResult4 instanceof DenseVector);
- Assert.assertArrayEquals(
- ((DenseVector) minusResult1).getData(), new double[] {0, 2, 2, 4}, TOL);
- Assert.assertArrayEquals(
- ((DenseVector) minusResult2).getData(), new double[] {0, -2, -2, -4}, TOL);
- Assert.assertArrayEquals(((SparseVector) minusResult3).getIndices(), new int[] {0, 2});
- Assert.assertArrayEquals(
- ((SparseVector) minusResult3).getValues(), new double[] {0., 0.}, TOL);
- Assert.assertArrayEquals(
- ((DenseVector) minusResult4).getData(), new double[] {0, 0, 0, 0}, TOL);
- }
-
- @Test
- public void testDot() throws Exception {
- Assert.assertEquals(MatVecOp.dot(dv, sv), 4.0, TOL);
- Assert.assertEquals(MatVecOp.dot(sv, dv), 4.0, TOL);
- Assert.assertEquals(MatVecOp.dot(sv, sv), 2.0, TOL);
- Assert.assertEquals(MatVecOp.dot(dv, dv), 30.0, TOL);
- }
-
- @Test
- public void testSumAbsDiff() throws Exception {
- Assert.assertEquals(MatVecOp.sumAbsDiff(dv, sv), 8.0, TOL);
- Assert.assertEquals(MatVecOp.sumAbsDiff(sv, dv), 8.0, TOL);
- Assert.assertEquals(MatVecOp.sumAbsDiff(sv, sv), 0.0, TOL);
- Assert.assertEquals(MatVecOp.sumAbsDiff(dv, dv), 0.0, TOL);
- }
-
- @Test
- public void testSumSquaredDiff() throws Exception {
- Assert.assertEquals(MatVecOp.sumSquaredDiff(dv, sv), 24.0, TOL);
- Assert.assertEquals(MatVecOp.sumSquaredDiff(sv, dv), 24.0, TOL);
- Assert.assertEquals(MatVecOp.sumSquaredDiff(sv, sv), 0.0, TOL);
- Assert.assertEquals(MatVecOp.sumSquaredDiff(dv, dv), 0.0, TOL);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/SparseVectorTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/SparseVectorTest.java
deleted file mode 100644
index 0f68102..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/SparseVectorTest.java
+++ /dev/null
@@ -1,232 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.Map;
-import java.util.TreeMap;
-
-/** Test cases for SparseVector. */
-public class SparseVectorTest {
- private static final double TOL = 1.0e-6;
- private SparseVector v1 =
- new SparseVector(8, new int[] {1, 3, 5, 7}, new double[] {2.0, 2.0, 2.0, 2.0});
- private SparseVector v2 =
- new SparseVector(8, new int[] {3, 4, 5}, new double[] {1.0, 1.0, 1.0});
-
- @Test
- public void testConstructor() throws Exception {
- int[] indices = new int[] {3, 7, 2, 1};
- double[] values = new double[] {3.0, 7.0, 2.0, 1.0};
- Map<Integer, Double> map = new TreeMap<>();
- for (int i = 0; i < indices.length; i++) {
- map.put(indices[i], values[i]);
- }
- SparseVector v = new SparseVector(8, map);
- Assert.assertArrayEquals(v.getIndices(), new int[] {1, 2, 3, 7});
- Assert.assertArrayEquals(v.getValues(), new double[] {1, 2, 3, 7}, TOL);
- }
-
- @Test
- public void testSize() throws Exception {
- Assert.assertEquals(v1.size(), 8);
- }
-
- @Test
- public void testSet() throws Exception {
- SparseVector v = v1.clone();
- v.set(2, 2.0);
- v.set(3, 3.0);
- Assert.assertEquals(v.get(2), 2.0, TOL);
- Assert.assertEquals(v.get(3), 3.0, TOL);
- }
-
- @Test
- public void testAdd() throws Exception {
- SparseVector v = v1.clone();
- v.add(2, 2.0);
- v.add(3, 3.0);
- Assert.assertEquals(v.get(2), 2.0, TOL);
- Assert.assertEquals(v.get(3), 5.0, TOL);
- }
-
- @Test
- public void testPrefix() throws Exception {
- SparseVector prefixed = v1.prefix(0.2);
- Assert.assertArrayEquals(prefixed.getIndices(), new int[] {0, 2, 4, 6, 8});
- Assert.assertArrayEquals(prefixed.getValues(), new double[] {0.2, 2, 2, 2, 2}, 0);
- }
-
- @Test
- public void testAppend() throws Exception {
- SparseVector prefixed = v1.append(0.2);
- Assert.assertArrayEquals(prefixed.getIndices(), new int[] {1, 3, 5, 7, 8});
- Assert.assertArrayEquals(prefixed.getValues(), new double[] {2, 2, 2, 2, 0.2}, 0);
- }
-
- @Test
- public void testSortIndices() throws Exception {
- int n = 8;
- int[] indices = new int[] {7, 5, 3, 1};
- double[] values = new double[] {7, 5, 3, 1};
- v1 = new SparseVector(n, indices, values);
- Assert.assertArrayEquals(values, new double[] {1, 3, 5, 7}, 0.);
- Assert.assertArrayEquals(v1.getValues(), new double[] {1, 3, 5, 7}, 0.);
- Assert.assertArrayEquals(indices, new int[] {1, 3, 5, 7});
- Assert.assertArrayEquals(v1.getIndices(), new int[] {1, 3, 5, 7});
- }
-
- @Test
- public void testNormL2Square() throws Exception {
- Assert.assertEquals(v2.normL2Square(), 3.0, TOL);
- }
-
- @Test
- public void testMinus() throws Exception {
- Vector d = v2.minus(v1);
- Assert.assertEquals(d.get(0), 0.0, TOL);
- Assert.assertEquals(d.get(1), -2.0, TOL);
- Assert.assertEquals(d.get(2), 0.0, TOL);
- Assert.assertEquals(d.get(3), -1.0, TOL);
- Assert.assertEquals(d.get(4), 1.0, TOL);
- }
-
- @Test
- public void testPlus() throws Exception {
- Vector d = v1.plus(v2);
- Assert.assertEquals(d.get(0), 0.0, TOL);
- Assert.assertEquals(d.get(1), 2.0, TOL);
- Assert.assertEquals(d.get(2), 0.0, TOL);
- Assert.assertEquals(d.get(3), 3.0, TOL);
-
- DenseVector dv = DenseVector.ones(8);
- dv = dv.plus(v2);
- Assert.assertArrayEquals(dv.getData(), new double[] {1, 1, 1, 2, 2, 2, 1, 1}, TOL);
- }
-
- @Test
- public void testDot() throws Exception {
- Assert.assertEquals(v1.dot(v2), 4.0, TOL);
- }
-
- @Test
- public void testGet() throws Exception {
- Assert.assertEquals(v1.get(5), 2.0, TOL);
- Assert.assertEquals(v1.get(6), 0.0, TOL);
- }
-
- @Test
- public void testSlice() throws Exception {
- int n = 8;
- int[] indices = new int[] {1, 3, 5, 7};
- double[] values = new double[] {2.0, 3.0, 4.0, 5.0};
- SparseVector v = new SparseVector(n, indices, values);
-
- int[] indices1 = new int[] {5, 4, 3};
- SparseVector vec1 = v.slice(indices1);
- Assert.assertEquals(vec1.size(), 3);
- Assert.assertArrayEquals(vec1.getIndices(), new int[] {0, 2});
- Assert.assertArrayEquals(vec1.getValues(), new double[] {4.0, 3.0}, 0.0);
-
- int[] indices2 = new int[] {3, 5};
- SparseVector vec2 = v.slice(indices2);
- Assert.assertArrayEquals(vec2.getIndices(), new int[] {0, 1});
- Assert.assertArrayEquals(vec2.getValues(), new double[] {3.0, 4.0}, 0.0);
-
- int[] indices3 = new int[] {2, 4};
- SparseVector vec3 = v.slice(indices3);
- Assert.assertEquals(vec3.size(), 2);
- Assert.assertArrayEquals(vec3.getIndices(), new int[] {});
- Assert.assertArrayEquals(vec3.getValues(), new double[] {}, 0.0);
-
- int[] indices4 = new int[] {2, 2, 4, 4};
- SparseVector vec4 = v.slice(indices4);
- Assert.assertEquals(vec4.size(), 4);
- Assert.assertArrayEquals(vec4.getIndices(), new int[] {});
- Assert.assertArrayEquals(vec4.getValues(), new double[] {}, 0.0);
- }
-
- @Test
- public void testToDenseVector() throws Exception {
- int[] indices = new int[] {1, 3, 5};
- double[] values = new double[] {1.0, 3.0, 5.0};
- SparseVector v = new SparseVector(-1, indices, values);
- DenseVector dv = v.toDenseVector();
- Assert.assertEquals(dv.size(), 6);
- Assert.assertArrayEquals(dv.getData(), new double[] {0, 1, 0, 3, 0, 5}, TOL);
- }
-
- @Test
- public void testRemoveZeroValues() throws Exception {
- int[] indices = new int[] {1, 3, 5};
- double[] values = new double[] {0.0, 3.0, 0.0};
- SparseVector v = new SparseVector(6, indices, values);
- v.removeZeroValues();
- Assert.assertArrayEquals(v.getIndices(), new int[] {3});
- Assert.assertArrayEquals(v.getValues(), new double[] {3}, TOL);
- }
-
- @Test
- public void testOuter() throws Exception {
- DenseMatrix outerProduct = v1.outer(v2);
- Assert.assertEquals(outerProduct.numRows(), 8);
- Assert.assertEquals(outerProduct.numCols(), 8);
- Assert.assertArrayEquals(
- outerProduct.getRow(0), new double[] {0, 0, 0, 0, 0, 0, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(1), new double[] {0, 0, 0, 2, 2, 2, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(2), new double[] {0, 0, 0, 0, 0, 0, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(3), new double[] {0, 0, 0, 2, 2, 2, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(4), new double[] {0, 0, 0, 0, 0, 0, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(5), new double[] {0, 0, 0, 2, 2, 2, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(6), new double[] {0, 0, 0, 0, 0, 0, 0, 0}, TOL);
- Assert.assertArrayEquals(
- outerProduct.getRow(7), new double[] {0, 0, 0, 2, 2, 2, 0, 0}, TOL);
- }
-
- @Test
- public void testIterator() throws Exception {
- VectorIterator iterator = v1.iterator();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 1);
- Assert.assertEquals(iterator.getValue(), 2, 0);
- iterator.next();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 3);
- Assert.assertEquals(iterator.getValue(), 2, 0);
- iterator.next();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 5);
- Assert.assertEquals(iterator.getValue(), 2, 0);
- iterator.next();
- Assert.assertTrue(iterator.hasNext());
- Assert.assertEquals(iterator.getIndex(), 7);
- Assert.assertEquals(iterator.getValue(), 2, 0);
- iterator.next();
- Assert.assertFalse(iterator.hasNext());
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/VectorUtilTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/VectorUtilTest.java
deleted file mode 100644
index 26d0ac4..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/linalg/VectorUtilTest.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.linalg;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/** Test cases for VectorUtil. */
-public class VectorUtilTest {
- @Test
- public void testParseDenseAndToString() {
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- String str = VectorUtil.toString(vec);
- Assert.assertEquals(str, "1.0 2.0 -3.0");
- Assert.assertArrayEquals(vec.getData(), VectorUtil.parseDense(str).getData(), 0);
- }
-
- @Test
- public void testParseDenseWithSpace() {
- DenseVector vec1 = VectorUtil.parseDense("1 2 -3");
- DenseVector vec2 = VectorUtil.parseDense(" 1 2 -3 ");
- DenseVector vec = new DenseVector(new double[] {1, 2, -3});
- Assert.assertArrayEquals(vec1.getData(), vec.getData(), 0);
- Assert.assertArrayEquals(vec2.getData(), vec.getData(), 0);
- }
-
- @Test
- public void testSparseToString() {
- SparseVector v1 =
- new SparseVector(8, new int[] {1, 3, 5, 7}, new double[] {2.0, 2.0, 2.0, 2.0});
- Assert.assertEquals(VectorUtil.toString(v1), "$8$1:2.0 3:2.0 5:2.0 7:2.0");
- }
-
- @Test
- public void testParseSparse() {
- SparseVector vec1 = VectorUtil.parseSparse("0:1 2:-3");
- SparseVector vec3 = VectorUtil.parseSparse("$4$0:1 2:-3");
- SparseVector vec4 = VectorUtil.parseSparse("$4$");
- SparseVector vec5 = VectorUtil.parseSparse("");
- Assert.assertEquals(vec1.get(0), 1., 0.);
- Assert.assertEquals(vec1.get(2), -3., 0.);
- Assert.assertArrayEquals(vec3.toDenseVector().getData(), new double[] {1, 0, -3, 0}, 0);
- Assert.assertEquals(vec3.size(), 4);
- Assert.assertArrayEquals(vec4.toDenseVector().getData(), new double[] {0, 0, 0, 0}, 0);
- Assert.assertEquals(vec4.size(), 4);
- Assert.assertEquals(vec5.size(), -1);
- }
-
- @Test
- public void testParseAndToStringOfVector() {
- Vector sparse = VectorUtil.parseSparse("0:1 2:-3");
- Vector dense = VectorUtil.parseDense("1 0 -3");
-
- Assert.assertEquals(VectorUtil.toString(sparse), "0:1.0 2:-3.0");
- Assert.assertEquals(VectorUtil.toString(dense), "1.0 0.0 -3.0");
- Assert.assertTrue(VectorUtil.parse("$4$0:1 2:-3") instanceof SparseVector);
- Assert.assertTrue(VectorUtil.parse("1 0 -3") instanceof DenseVector);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussianTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussianTest.java
deleted file mode 100644
index 1bda854..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/statistics/basicstatistic/MultivariateGaussianTest.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.statistics.basicstatistic;
-
-import org.apache.flink.ml.common.linalg.DenseMatrix;
-import org.apache.flink.ml.common.linalg.DenseVector;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/** Test cases for {@link MultivariateGaussian}. */
-public class MultivariateGaussianTest {
- private static final double TOL = 1.0e-5;
-
- @Test
- public void testUnivariate() throws Exception {
- DenseVector x1 = new DenseVector(new double[] {0.0});
- DenseVector x2 = new DenseVector(new double[] {1.5});
- DenseVector mu = DenseVector.zeros(1);
- DenseMatrix sigma1 = DenseMatrix.ones(1, 1);
- MultivariateGaussian dist1 = new MultivariateGaussian(mu, sigma1);
- Assert.assertEquals(dist1.pdf(x1), 0.39894, TOL);
- Assert.assertEquals(dist1.pdf(x2), 0.12952, TOL);
-
- DenseMatrix sigma2 = DenseMatrix.ones(1, 1);
- sigma2.scaleEqual(4.0);
- MultivariateGaussian dist2 = new MultivariateGaussian(mu, sigma2);
- Assert.assertEquals(dist2.pdf(x1), 0.19947, TOL);
- Assert.assertEquals(dist2.pdf(x2), 0.15057, TOL);
- }
-
- @Test
- public void testMultivariate() throws Exception {
- DenseVector mu = DenseVector.zeros(2);
-
- DenseMatrix sigma1 = DenseMatrix.eye(2);
- MultivariateGaussian mg1 = new MultivariateGaussian(mu, sigma1);
- Assert.assertEquals(mg1.pdf(DenseVector.zeros(2)), 0.15915, TOL);
- Assert.assertEquals(mg1.pdf(DenseVector.ones(2)), 0.05855, TOL);
-
- DenseMatrix sigma2 = new DenseMatrix(2, 2, new double[] {4.0, -1.0, -1.0, 2.0});
- MultivariateGaussian mg2 = new MultivariateGaussian(mu, sigma2);
- Assert.assertEquals(mg2.pdf(DenseVector.zeros(2)), 0.060155, TOL);
- Assert.assertEquals(mg2.pdf(DenseVector.ones(2)), 0.033971, TOL);
- }
-
- @Test
- public void testMultivariateDegenerate() throws Exception {
- DenseVector mu = DenseVector.zeros(2);
- DenseMatrix sigma = new DenseMatrix(2, 2, new double[] {1.0, 1.0, 1.0, 1.0});
- MultivariateGaussian mg = new MultivariateGaussian(mu, sigma);
- Assert.assertEquals(mg.pdf(DenseVector.zeros(2)), 0.11254, TOL);
- Assert.assertEquals(mg.pdf(DenseVector.ones(2)), 0.068259, TOL);
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataSetConversionUtilTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataSetConversionUtilTest.java
deleted file mode 100644
index 4b84569..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataSetConversionUtilTest.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.ml.common.MLEnvironmentFactory;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.types.Row;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.ExecutionException;
-
-/** Unit test for DataSetConversionUtil. */
-public class DataSetConversionUtilTest {
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testForceTypeSchema() {
- ExecutionEnvironment env = MLEnvironmentFactory.getDefault().getExecutionEnvironment();
-
- DataSet<Row> input = env.fromElements(Row.of("s1")).map(new GenericTypeMap());
- Table table2 =
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)});
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table2.getSchema());
- }
-
- @Test
- public void testForceTypeWithTableSchema() {
- ExecutionEnvironment env = MLEnvironmentFactory.getDefault().getExecutionEnvironment();
-
- DataSet<Row> input = env.fromElements(Row.of("s1")).map(new GenericTypeMap());
- Table table2 =
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}));
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table2.getSchema());
- }
-
- @Test
- public void testExceptionWithoutTypeSchema() {
- thrown.expect(ValidationException.class);
- ExecutionEnvironment env = MLEnvironmentFactory.getDefault().getExecutionEnvironment();
- DataSet<Row> input = env.fromElements(Row.of("s1")).map(new GenericTypeMap());
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, input, new String[] {"f0"});
- }
-
- @Test
- public void testBasicConvert() throws Exception {
- ExecutionEnvironment env = MLEnvironmentFactory.getDefault().getExecutionEnvironment();
-
- DataSet<Row> input = env.fromElements(Row.of("a"));
-
- Table table1 =
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new String[] {"word"});
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(String.class)}),
- table1.getSchema());
- List<Row> list =
- DataSetConversionUtil.fromTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, table1)
- .collect();
- Assert.assertEquals(Collections.singletonList(Row.of("a")), list);
- }
-
- @Test
- public void testE2E() throws Exception {
- ExecutionEnvironment env = MLEnvironmentFactory.getDefault().getExecutionEnvironment();
-
- DataSet<Row> input = env.fromElements(Row.of("a"));
-
- Table table1 =
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new String[] {"word"});
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(String.class)}),
- table1.getSchema());
-
- DataSet<Row> genericInput1 = input.map(new GenericTypeMap());
-
- // Force type should go through with explicit type info.
- Table table2 =
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput1,
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)});
-
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table2.getSchema());
-
- DataSet<Row> genericInput2 = input.map(new GenericTypeMap());
-
- // Force type should go through with table schema.
- Table table3 =
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput2,
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}));
-
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table3.getSchema());
-
- // applying toTable again on the same input should fail
- thrown.expect(IllegalStateException.class);
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput2,
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}));
-
- // Validation should fail without correct type inference.
- DataSet<Row> genericInput3 = input.map(new GenericTypeMap());
- thrown.expect(ValidationException.class);
- DataSetConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput3,
- new String[] {"word"});
-
- // Output should go through when using correct type to output.
- DataSet<Row> output =
- DataSetConversionUtil.fromTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, table1);
- output.print();
-
- // Output should NOT go through when using incorrect type forcing.
- thrown.expect(ExecutionException.class);
- DataSetConversionUtil.fromTable(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, table2)
- .print();
- }
-
- private static class GenericTypeMap implements MapFunction<Row, Row> {
-
- @Override
- public Row map(Row value) throws Exception {
- return value;
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataStreamConversionUtilTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataStreamConversionUtilTest.java
deleted file mode 100644
index 60edbc1..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/DataStreamConversionUtilTest.java
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.ml.common.MLEnvironmentFactory;
-import org.apache.flink.streaming.api.datastream.DataStream;
-import org.apache.flink.streaming.api.datastream.DataStreamUtils;
-import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
-import org.apache.flink.table.api.Table;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.table.api.ValidationException;
-import org.apache.flink.types.Row;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.Iterator;
-import java.util.concurrent.ExecutionException;
-
-/** Unit Test for DataStreamConversionUtil. */
-public class DataStreamConversionUtilTest {
- @Rule public ExpectedException thrown = ExpectedException.none();
-
- @Test
- public void testForceTypeSchema() {
- StreamExecutionEnvironment env =
- MLEnvironmentFactory.getDefault().getStreamExecutionEnvironment();
-
- DataStream<Row> input =
- env.fromElements(Row.of("s1"))
- .map(new DataStreamConversionUtilTest.GenericTypeMap());
- Table table2 =
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)});
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table2.getSchema());
- }
-
- @Test
- public void testForceTypeWithTableSchema() {
- StreamExecutionEnvironment env =
- MLEnvironmentFactory.getDefault().getStreamExecutionEnvironment();
-
- DataStream<Row> input =
- env.fromElements(Row.of("s1"))
- .map(new DataStreamConversionUtilTest.GenericTypeMap());
- Table table2 =
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}));
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table2.getSchema());
- }
-
- @Test
- public void testExceptionWithoutTypeSchema() {
- thrown.expect(ValidationException.class);
- StreamExecutionEnvironment env =
- MLEnvironmentFactory.getDefault().getStreamExecutionEnvironment();
- DataStream<Row> input =
- env.fromElements(Row.of("s1"))
- .map(new DataStreamConversionUtilTest.GenericTypeMap());
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, input, new String[] {"f0"});
- }
-
- @Test
- public void testBasicConvert() throws Exception {
- StreamExecutionEnvironment env =
- MLEnvironmentFactory.getDefault().getStreamExecutionEnvironment();
- DataStream<Row> input = env.fromElements(Row.of("a"));
- Table table1 =
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new String[] {"word"});
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(String.class)}),
- table1.getSchema());
- DataStream<Row> rowDataStream =
- DataStreamConversionUtil.fromTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, table1);
- Iterator<Row> result = DataStreamUtils.collect(rowDataStream);
- Assert.assertEquals(Row.of("a"), result.next());
- Assert.assertFalse(result.hasNext());
- }
-
- @Test
- public void testE2E() throws Exception {
- StreamExecutionEnvironment env =
- MLEnvironmentFactory.getDefault().getStreamExecutionEnvironment();
-
- DataStream<Row> input = env.fromElements(Row.of("a"));
-
- Table table1 =
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- input,
- new String[] {"word"});
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(String.class)}),
- table1.getSchema());
-
- DataStream<Row> genericInput1 = input.map(new GenericTypeMap());
-
- // Force type should go through with explicit type info on generic type input.
- Table table2 =
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput1,
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)});
-
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table2.getSchema());
-
- DataStream<Row> genericInput2 = input.map(new GenericTypeMap());
-
- // Force type should go through with table schema on generic type input.
- Table table3 =
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput2,
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}));
-
- Assert.assertEquals(
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}),
- table3.getSchema());
-
- // applying toTable again on the same input should fail
- thrown.expect(IllegalStateException.class);
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput2,
- new TableSchema(
- new String[] {"word"},
- new TypeInformation[] {TypeInformation.of(Integer.class)}));
-
- // Validation should fail due to type inference error.
- DataStream<Row> genericInput3 = input.map(new GenericTypeMap());
- thrown.expect(ValidationException.class);
- DataStreamConversionUtil.toTable(
- MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID,
- genericInput3,
- new String[] {"word"});
-
- // Output should go through when using correct type to output.
- DataStreamConversionUtil.fromTable(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, table1)
- .print();
-
- // Output should NOT go through when using incorrect type forcing.
- thrown.expect(ExecutionException.class);
- DataStreamConversionUtil.fromTable(MLEnvironmentFactory.DEFAULT_ML_ENVIRONMENT_ID, table2)
- .print();
- }
-
- private static class GenericTypeMap implements MapFunction<Row, Row> {
-
- @Override
- public Row map(Row value) throws Exception {
- return value;
- }
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/OutputColsHelperTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/OutputColsHelperTest.java
deleted file mode 100644
index 8b89e1e9..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/OutputColsHelperTest.java
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.types.Row;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/** Unit test for OutputColsHelper. */
-public class OutputColsHelperTest {
-
- private TableSchema tableSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(Integer.class)
- });
- private String[] reservedColNames = new String[] {"f0"};
- private Row row = Row.of("a", 1L, 1);
-
- @Test
- public void testResultSchema() {
- TableSchema expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(Integer.class),
- TypeInformation.of(String.class)
- });
- OutputColsHelper helper =
- new OutputColsHelper(tableSchema, "res", TypeInformation.of(String.class));
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class), TypeInformation.of(String.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema, "res", TypeInformation.of(String.class), reservedColNames);
- Assert.assertEquals(expectSchema, helper.getResultSchema());
- Assert.assertArrayEquals(reservedColNames, helper.getReservedColumns());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "res1", "res2"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(String.class),
- TypeInformation.of(Integer.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res1", "res2"},
- new TypeInformation[] {
- TypeInformation.of(String.class), TypeInformation.of(Integer.class)
- },
- reservedColNames);
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(Integer.class),
- TypeInformation.of(String.class)
- });
- helper = new OutputColsHelper(tableSchema, "res", TypeInformation.of(String.class));
- Assert.assertEquals(expectSchema, helper.getResultSchema());
- Assert.assertArrayEquals(tableSchema.getFieldNames(), helper.getReservedColumns());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(Integer.class)
- });
- helper = new OutputColsHelper(tableSchema, "f0", TypeInformation.of(Integer.class));
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(String.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"f0", "f2"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class), TypeInformation.of(String.class)
- });
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class), TypeInformation.of(Integer.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res", "f0"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class), TypeInformation.of(String.class)
- },
- reservedColNames);
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(Integer.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class), TypeInformation.of(String.class)
- },
- new String[] {"f1", "f0"});
- Assert.assertEquals(expectSchema, helper.getResultSchema());
- }
-
- @Test
- public void testResultRow() {
- OutputColsHelper helper =
- new OutputColsHelper(tableSchema, "res", TypeInformation.of(String.class));
- Row expectRow = Row.of("a", 1L, 1, "b");
- Assert.assertEquals(helper.getResultRow(row, Row.of("b")), expectRow);
-
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res1", "res2"},
- new TypeInformation[] {
- TypeInformation.of(String.class), TypeInformation.of(Integer.class)
- });
- expectRow = Row.of("a", 1L, 1, "b", 2);
- Assert.assertEquals(expectRow, helper.getResultRow(row, Row.of("b", 2)));
-
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res", "f0"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class), TypeInformation.of(String.class)
- },
- reservedColNames);
- expectRow = Row.of("b", 2);
- Assert.assertEquals(expectRow, helper.getResultRow(row, Row.of(2, "b")));
- }
-
- @Test
- public void testExceptionCase() {
- TableSchema expectSchema =
- new TableSchema(
- new String[] {"f0", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class), TypeInformation.of(Integer.class)
- });
- OutputColsHelper helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res", "f0"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class), TypeInformation.of(String.class)
- },
- new String[] {"res", "res2"});
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "res"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(Long.class),
- TypeInformation.of(Integer.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"res", "f0"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class), TypeInformation.of(String.class)
- },
- new String[] {"f1", "res"});
- Assert.assertEquals(expectSchema, helper.getResultSchema());
-
- expectSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2"},
- new TypeInformation[] {
- TypeInformation.of(String.class),
- TypeInformation.of(Integer.class),
- TypeInformation.of(Double.class)
- });
- helper =
- new OutputColsHelper(
- tableSchema,
- new String[] {"f1", "f0", "f2"},
- new TypeInformation[] {
- TypeInformation.of(Integer.class),
- TypeInformation.of(String.class),
- TypeInformation.of(Double.class)
- },
- new String[] {"f1", "res"});
- Assert.assertEquals(expectSchema, helper.getResultSchema());
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/TableUtilTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/TableUtilTest.java
deleted file mode 100644
index e3733c3..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/TableUtilTest.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.typeinfo.TypeInformation;
-import org.apache.flink.api.common.typeinfo.Types;
-import org.apache.flink.table.api.TableSchema;
-import org.apache.flink.types.Row;
-
-import org.junit.Assert;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import java.util.Collections;
-
-/** Unit test for TableUtil. */
-public class TableUtilTest {
- @Rule public ExpectedException thrown = ExpectedException.none();
- private String[] colNames = new String[] {"f0", "f1", "f2"};
- private TableSchema tableSchema =
- new TableSchema(colNames, new TypeInformation[] {Types.INT, Types.LONG, Types.STRING});
-
- @Test
- public void testFindIndexFromName() {
- String[] colNames = new String[] {"f0", "f1", "F2"};
- Assert.assertEquals(0, TableUtil.findColIndex(colNames, "f0"));
- Assert.assertEquals(1, TableUtil.findColIndex(colNames, "F1"));
- Assert.assertEquals(-1, TableUtil.findColIndex(colNames, "f3"));
- Assert.assertEquals(0, TableUtil.findColIndex(tableSchema, "f0"));
-
- Assert.assertArrayEquals(
- new int[] {1, 2}, TableUtil.findColIndices(colNames, new String[] {"f1", "F2"}));
- Assert.assertArrayEquals(
- new int[] {1, 2}, TableUtil.findColIndices(tableSchema, new String[] {"f1", "F2"}));
- Assert.assertArrayEquals(
- new int[] {-1, 2},
- TableUtil.findColIndices(tableSchema, new String[] {"f3", "F2"}));
- Assert.assertArrayEquals(new int[] {0, 1, 2}, TableUtil.findColIndices(colNames, null));
- }
-
- @Test
- public void testFindTypeFromTable() {
- Assert.assertArrayEquals(
- new TypeInformation[] {TypeInformation.of(Integer.class), Types.LONG},
- TableUtil.findColTypes(tableSchema, new String[] {"f0", "f1"}));
- Assert.assertArrayEquals(
- new TypeInformation[] {Types.LONG, null},
- TableUtil.findColTypes(tableSchema, new String[] {"f1", "f3"}));
- Assert.assertArrayEquals(
- new TypeInformation[] {Types.INT, Types.LONG, Types.STRING},
- TableUtil.findColTypes(tableSchema, null));
-
- Assert.assertEquals(
- TypeInformation.of(Integer.class), TableUtil.findColType(tableSchema, "f0"));
- Assert.assertNull(TableUtil.findColType(tableSchema, "f3"));
- }
-
- @Test
- public void isNumberIsStringTest() {
- Assert.assertTrue(TableUtil.isSupportedNumericType(Types.INT));
- Assert.assertTrue(TableUtil.isSupportedNumericType(Types.DOUBLE));
- Assert.assertTrue(TableUtil.isSupportedNumericType(Types.LONG));
- Assert.assertTrue(TableUtil.isSupportedNumericType(Types.BYTE));
- Assert.assertTrue(TableUtil.isSupportedNumericType(Types.FLOAT));
- Assert.assertTrue(TableUtil.isSupportedNumericType(Types.SHORT));
- Assert.assertFalse(TableUtil.isSupportedNumericType(Types.STRING));
- Assert.assertTrue(TableUtil.isString(Types.STRING));
- }
-
- @Test
- public void assertColExistOrTypeTest() {
- String[] colNames = new String[] {"f0", "f1", "f2"};
- TableUtil.assertSelectedColExist(colNames, null);
- TableUtil.assertSelectedColExist(colNames, "f0");
- TableUtil.assertSelectedColExist(colNames, "f0", "f1");
-
- TableUtil.assertNumericalCols(tableSchema, null);
- TableUtil.assertNumericalCols(tableSchema, "f1");
- TableUtil.assertNumericalCols(tableSchema, "f0", "f1");
-
- TableUtil.assertStringCols(tableSchema, null);
- TableUtil.assertStringCols(tableSchema, "f2");
-
- TableUtil.assertVectorCols(tableSchema, null);
- }
-
- @Test
- public void assertColExistOrTypeExceptionTest() {
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage(" col is not exist f3");
- TableUtil.assertSelectedColExist(colNames, "f3");
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage(" col is not exist f3");
- TableUtil.assertSelectedColExist(colNames, "f0", "f3");
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("col type must be number f2");
- TableUtil.assertNumericalCols(tableSchema, "f2");
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("col type must be number f2");
- TableUtil.assertNumericalCols(tableSchema, "f2", "f0");
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("col type must be string f2");
- TableUtil.assertStringCols(tableSchema, "f2");
-
- thrown.expect(IllegalArgumentException.class);
- thrown.expectMessage("col type must be string f0");
- TableUtil.assertStringCols(tableSchema, "f0", "f3");
- }
-
- @Test
- public void getNumericColsTest() {
- TableSchema tableSchema =
- new TableSchema(
- new String[] {"f0", "f1", "F2", "f3"},
- new TypeInformation[] {Types.INT, Types.LONG, Types.STRING, Types.BOOLEAN});
-
- Assert.assertArrayEquals(new String[] {"f0", "f1"}, TableUtil.getNumericCols(tableSchema));
- Assert.assertArrayEquals(
- new String[] {"f1"}, TableUtil.getNumericCols(tableSchema, new String[] {"f0"}));
- Assert.assertArrayEquals(
- new String[] {"f0", "f1"},
- TableUtil.getNumericCols(tableSchema, new String[] {"f2"}));
- }
-
- @Test
- public void getCategoricalColsTest() {
- TableSchema tableSchema =
- new TableSchema(
- new String[] {"f0", "f1", "f2", "f3"},
- new TypeInformation[] {Types.INT, Types.LONG, Types.STRING, Types.BOOLEAN});
-
- Assert.assertArrayEquals(
- new String[] {"f2", "f3"},
- TableUtil.getCategoricalCols(tableSchema, tableSchema.getFieldNames(), null));
- Assert.assertArrayEquals(
- new String[] {"f2", "f0", "f3"},
- TableUtil.getCategoricalCols(
- tableSchema, new String[] {"f2", "f1", "f0", "f3"}, new String[] {"f0"}));
-
- thrown.expect(IllegalArgumentException.class);
- Assert.assertArrayEquals(
- new String[] {"f3", "f2"},
- TableUtil.getCategoricalCols(
- tableSchema, new String[] {"f3", "f0"}, new String[] {"f2"}));
- }
-
- @Test
- public void getStringColsTest() {
- TableSchema tableSchema =
- new TableSchema(
- new String[] {"f0", "f1", "F2", "f3"},
- new TypeInformation[] {Types.INT, Types.LONG, Types.STRING, Types.BOOLEAN});
-
- Assert.assertArrayEquals(new String[] {"F2"}, TableUtil.getStringCols(tableSchema));
- Assert.assertArrayEquals(
- new String[] {}, TableUtil.getStringCols(tableSchema, new String[] {"F2"}));
- }
-
- @Test
- public void formatTest() {
- TableSchema tableSchema =
- new TableSchema(
- new String[] {"f0", "f1", "F2", "f3"},
- new TypeInformation[] {Types.INT, Types.LONG, Types.STRING, Types.BOOLEAN});
- Row row = Row.of(1, 2L, "3", true);
-
- String format =
- TableUtil.format(tableSchema.getFieldNames(), Collections.singletonList(row));
- Assert.assertTrue(
- ("f0|f1|F2|f3\r\n" + "--|--|--|--\n" + "1|2|3|true").equalsIgnoreCase(format));
- }
-
- @Test
- public void testTempTable() {
- Assert.assertTrue(TableUtil.getTempTableName().startsWith("temp_"));
- Assert.assertFalse(TableUtil.getTempTableName().contains("-"));
- }
-}
diff --git a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/VectorTypesTest.java b/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/VectorTypesTest.java
deleted file mode 100644
index 054956d..0000000
--- a/flink-ml-parent/flink-ml-lib/src/test/java/org/apache/flink/ml/common/utils/VectorTypesTest.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.flink.ml.common.utils;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.core.memory.DataInputDeserializer;
-import org.apache.flink.core.memory.DataOutputSerializer;
-import org.apache.flink.ml.common.linalg.DenseVector;
-import org.apache.flink.ml.common.linalg.SparseVector;
-import org.apache.flink.ml.common.linalg.Vector;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.concurrent.ThreadLocalRandom;
-
-/** Test cases for VectorTypes. */
-public class VectorTypesTest {
- @SuppressWarnings("unchecked")
- private static <V extends Vector> void doVectorSerDeserTest(TypeSerializer ser, V vector)
- throws IOException {
- DataOutputSerializer out = new DataOutputSerializer(1024);
- ser.serialize(vector, out);
- DataInputDeserializer in = new DataInputDeserializer(out.getCopyOfBuffer());
- Vector deserialize = (Vector) ser.deserialize(in);
- Assert.assertEquals(vector.getClass(), deserialize.getClass());
- Assert.assertEquals(vector, deserialize);
- }
-
- @Test
- public void testVectorsSerDeser() throws IOException {
- // Prepare data
- SparseVector sparseVector =
- new SparseVector(
- 10,
- new HashMap<Integer, Double>() {
- {
- ThreadLocalRandom rand = ThreadLocalRandom.current();
- for (int i = 0; i < 10; i += 2) {
- this.put(i, rand.nextDouble());
- }
- }
- });
- DenseVector denseVector = DenseVector.rand(10);
-
- // Prepare serializer
- ExecutionConfig config = new ExecutionConfig();
- TypeSerializer<Vector> vecSer = VectorTypes.VECTOR.createSerializer(config);
... 540 lines suppressed ...
[flink] 05/05: [FLINK-21976] Remove pyflink/ml related configs from
python scripts
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit 75e1f60ad741f87af3e15c5ab3633ce7e4c17643
Author: Dong Lin <li...@gmail.com>
AuthorDate: Thu Apr 1 00:11:27 2021 +0800
[FLINK-21976] Remove pyflink/ml related configs from python scripts
This closes #15405.
---
flink-python/README.md | 2 +-
flink-python/docs/index.rst | 1 -
flink-python/docs/pyflink.ml.rst | 52 -------------------------
flink-python/pyflink/shell.py | 2 -
flink-python/pyflink/testing/test_case_utils.py | 34 ----------------
flink-python/setup.py | 7 +---
flink-python/tox.ini | 2 +-
7 files changed, 3 insertions(+), 97 deletions(-)
diff --git a/flink-python/README.md b/flink-python/README.md
index d14aaab..e5fbf95 100644
--- a/flink-python/README.md
+++ b/flink-python/README.md
@@ -16,7 +16,7 @@ The auto-generated Python docs can be found at [https://ci.apache.org/projects/f
## Python Requirements
-Apache Flink Python API depends on Py4J (currently version 0.10.8.1), CloudPickle (currently version 1.2.2), python-dateutil(currently version 2.8.0), Apache Beam (currently version 2.27.0) and jsonpickle (currently 1.2).
+Apache Flink Python API depends on Py4J (currently version 0.10.8.1), CloudPickle (currently version 1.2.2), python-dateutil(currently version 2.8.0), Apache Beam (currently version 2.27.0).
## Development Notices
diff --git a/flink-python/docs/index.rst b/flink-python/docs/index.rst
index 4da7ee1..3d6f538 100644
--- a/flink-python/docs/index.rst
+++ b/flink-python/docs/index.rst
@@ -28,7 +28,6 @@ Welcome to Flink Python API Docs!
pyflink.table
pyflink.dataset
pyflink.datastream
- pyflink.ml
pyflink.metrics
diff --git a/flink-python/docs/pyflink.ml.rst b/flink-python/docs/pyflink.ml.rst
deleted file mode 100644
index 5b399b3..0000000
--- a/flink-python/docs/pyflink.ml.rst
+++ /dev/null
@@ -1,52 +0,0 @@
-.. ################################################################################
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- ################################################################################
-
-pyflink.ml package
-=====================
-
-Module contents
----------------
-
-.. automodule:: pyflink.ml
- :members:
- :undoc-members:
- :show-inheritance:
-
-pyflink.ml.api module
----------------------------
-.. automodule:: pyflink.ml.api
- :members:
- :undoc-members:
-
-pyflink.ml.api.param module
----------------------------
-.. automodule:: pyflink.ml.api.param
- :members:
- :undoc-members:
-
-pyflink.ml.lib module
--------------------------------------
-.. automodule:: pyflink.ml.lib
- :members:
- :undoc-members:
-
-pyflink.ml.lib.param module
----------------------------
-.. automodule:: pyflink.ml.lib.param
- :members:
- :undoc-members:
diff --git a/flink-python/pyflink/shell.py b/flink-python/pyflink/shell.py
index d7f353a..b5c9dc0 100644
--- a/flink-python/pyflink/shell.py
+++ b/flink-python/pyflink/shell.py
@@ -27,8 +27,6 @@ from pyflink.table.catalog import *
from pyflink.table.descriptors import *
from pyflink.table.window import *
from pyflink.metrics import *
-from pyflink.ml.api import *
-from pyflink.ml.lib import *
utf8_out = open(sys.stdout.fileno(), mode='w', encoding='utf8', buffering=1)
diff --git a/flink-python/pyflink/testing/test_case_utils.py b/flink-python/pyflink/testing/test_case_utils.py
index 479b53d..e086ca4 100644
--- a/flink-python/pyflink/testing/test_case_utils.py
+++ b/flink-python/pyflink/testing/test_case_utils.py
@@ -399,37 +399,3 @@ class TestEnv(object):
for item in self.result:
result[item.f0] = item.f1
return result
-
-
-class MLTestCase(PyFlinkTestCase):
- """
- Base class for testing ML.
- """
-
- _inited = False
-
- @staticmethod
- def _ensure_path(pattern):
- if not glob.glob(pattern):
- raise unittest.SkipTest(
- "'%s' is not available. Will skip the related tests." % pattern)
-
- @classmethod
- def _ensure_initialized(cls):
- if MLTestCase._inited:
- return
-
- flink_source_root_dir = _find_flink_source_root()
- api_path_pattern = (
- "flink-ml-parent/flink-ml-api/target/flink-ml-api*-SNAPSHOT.jar")
- lib_path_pattern = (
- "flink-ml-parent/flink-ml-lib/target/flink-ml-lib*-SNAPSHOT.jar")
-
- MLTestCase._ensure_path(os.path.join(flink_source_root_dir, api_path_pattern))
- MLTestCase._ensure_path(os.path.join(flink_source_root_dir, lib_path_pattern))
-
- MLTestCase._inited = True
-
- def setUp(self):
- super(MLTestCase, self).setUp()
- MLTestCase._ensure_initialized()
diff --git a/flink-python/setup.py b/flink-python/setup.py
index 12b5c40..8bb7229 100644
--- a/flink-python/setup.py
+++ b/flink-python/setup.py
@@ -297,11 +297,6 @@ try:
'pyflink.fn_execution.table',
'pyflink.fn_execution.utils',
'pyflink.metrics',
- 'pyflink.ml',
- 'pyflink.ml.api',
- 'pyflink.ml.api.param',
- 'pyflink.ml.lib',
- 'pyflink.ml.lib.param',
'pyflink.conf',
'pyflink.log',
'pyflink.examples',
@@ -340,7 +335,7 @@ try:
python_requires='>=3.6',
install_requires=['py4j==0.10.8.1', 'python-dateutil==2.8.0', 'apache-beam==2.27.0',
'cloudpickle==1.2.2', 'avro-python3>=1.8.1,!=1.9.2,<1.10.0',
- 'jsonpickle==1.2', 'pandas>=1.0,<1.2.0', 'pyarrow>=0.15.1,<3.0.0',
+ 'pandas>=1.0,<1.2.0', 'pyarrow>=0.15.1,<3.0.0',
'pytz>=2018.3', 'numpy>=1.14.3,<1.20', 'fastavro>=0.21.4,<0.24',
apache_flink_libraries_dependency],
cmdclass={'build_ext': build_ext},
diff --git a/flink-python/tox.ini b/flink-python/tox.ini
index 34b8c5f..61b7449 100644
--- a/flink-python/tox.ini
+++ b/flink-python/tox.ini
@@ -53,7 +53,7 @@ max-line-length=100
exclude=.tox/*,dev/*,lib/*,target/*,build/*,dist/*,pyflink/shell.py,.eggs/*,pyflink/fn_execution/tests/process_mode_test_data.py,pyflink/fn_execution/*_pb2.py
[mypy]
-files=pyflink/common/*.py,pyflink/table/*.py,pyflink/dataset/*.py,pyflink/datastream/*.py,pyflink/ml/*.py,pyflink/metrics/*.py
+files=pyflink/common/*.py,pyflink/table/*.py,pyflink/dataset/*.py,pyflink/datastream/*.py,pyflink/metrics/*.py
ignore_missing_imports = True
strict_optional=False
[flink] 04/05: [FLINK-21976] Remove python packages under
flink-python/pyflink/ml
Posted by di...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
commit b6b16ebf28f142a06ce5fe550258e71486e614a2
Author: Dong Lin <li...@gmail.com>
AuthorDate: Thu Apr 1 00:10:29 2021 +0800
[FLINK-21976] Remove python packages under flink-python/pyflink/ml
This closes #15405.
---
flink-python/pyflink/ml/__init__.py | 17 -
flink-python/pyflink/ml/api/__init__.py | 28 --
flink-python/pyflink/ml/api/base.py | 348 --------------------
flink-python/pyflink/ml/api/ml_environment.py | 92 ------
.../pyflink/ml/api/ml_environment_factory.py | 125 --------
flink-python/pyflink/ml/api/param/__init__.py | 21 --
flink-python/pyflink/ml/api/param/base.py | 350 ---------------------
flink-python/pyflink/ml/lib/__init__.py | 17 -
flink-python/pyflink/ml/lib/param/__init__.py | 21 --
flink-python/pyflink/ml/lib/param/colname.py | 78 -----
flink-python/pyflink/ml/tests/__init__.py | 17 -
.../pyflink/ml/tests/test_ml_environment.py | 55 ----
.../ml/tests/test_ml_environment_factory.py | 48 ---
flink-python/pyflink/ml/tests/test_params.py | 187 -----------
flink-python/pyflink/ml/tests/test_pipeline.py | 171 ----------
.../pyflink/ml/tests/test_pipeline_it_case.py | 189 -----------
.../pyflink/ml/tests/test_pipeline_stage.py | 90 ------
17 files changed, 1854 deletions(-)
diff --git a/flink-python/pyflink/ml/__init__.py b/flink-python/pyflink/ml/__init__.py
deleted file mode 100644
index 65b48d4..0000000
--- a/flink-python/pyflink/ml/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
diff --git a/flink-python/pyflink/ml/api/__init__.py b/flink-python/pyflink/ml/api/__init__.py
deleted file mode 100644
index 90b3eb8..0000000
--- a/flink-python/pyflink/ml/api/__init__.py
+++ /dev/null
@@ -1,28 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.api.ml_environment import MLEnvironment
-from pyflink.ml.api.ml_environment_factory import MLEnvironmentFactory
-from pyflink.ml.api.base import Transformer, Estimator, Model, Pipeline, \
- PipelineStage, JavaTransformer, JavaEstimator, JavaModel
-
-
-__all__ = [
- "MLEnvironment", "MLEnvironmentFactory", "Transformer", "Estimator", "Model",
- "Pipeline", "PipelineStage", "JavaTransformer", "JavaEstimator", "JavaModel"
-]
diff --git a/flink-python/pyflink/ml/api/base.py b/flink-python/pyflink/ml/api/base.py
deleted file mode 100644
index 2db1249..0000000
--- a/flink-python/pyflink/ml/api/base.py
+++ /dev/null
@@ -1,348 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-import re
-
-from abc import ABCMeta, abstractmethod
-
-from pyflink.table.table_environment import TableEnvironment
-from pyflink.table.table import Table
-from pyflink.ml.api.param import WithParams, Params
-from pyflink.java_gateway import get_gateway
-from py4j.java_gateway import get_field, get_java_class as java_class
-
-
-class PipelineStage(WithParams):
- """
- Base class for a stage in a pipeline. The interface is only a concept, and does not have any
- actual functionality. Its subclasses must be either Estimator or Transformer. No other classes
- should inherit this interface directly.
-
- Each pipeline stage is with parameters, and requires a public empty constructor for
- restoration in Pipeline.
-
- .. versionadded:: 1.11.0
- """
-
- def __init__(self, params=None):
- if params is None:
- self._params = Params()
- else:
- self._params = params
-
- def get_params(self) -> Params:
- return self._params
-
- def _convert_params_to_java(self, j_pipeline_stage):
- for param in self._params._param_map:
- java_param = self._make_java_param(j_pipeline_stage, param)
- java_value = self._make_java_value(self._params._param_map[param])
- j_pipeline_stage.set(java_param, java_value)
-
- @staticmethod
- def _make_java_param(j_pipeline_stage, param):
- # camel case to snake case
- name = re.sub(r'(?<!^)(?=[A-Z])', '_', param.name).upper()
- return get_field(j_pipeline_stage, name)
-
- @staticmethod
- def _make_java_value(obj):
- """ Convert Python object into Java """
- if isinstance(obj, list):
- obj = [PipelineStage._make_java_value(x) for x in obj]
- return obj
-
- def to_json(self) -> str:
- return self.get_params().to_json()
-
- def load_json(self, json: str) -> None:
- self.get_params().load_json(json)
-
-
-class Transformer(PipelineStage, metaclass=ABCMeta):
- """
- A transformer is a PipelineStage that transforms an input Table to a result Table.
-
- .. versionadded:: 1.11.0
- """
-
- @abstractmethod
- def transform(self, table_env: TableEnvironment, table: Table) -> Table:
- """
- Applies the transformer on the input table, and returns the result table.
-
- :param table_env: the table environment to which the input table is bound.
- :param table: the table to be transformed
- :returns: the transformed table
-
- .. versionadded:: 1.11.0
- """
- raise NotImplementedError()
-
-
-class JavaTransformer(Transformer):
- """
- Base class for Transformer that wrap Java implementations. Subclasses should
- ensure they have the transformer Java object available as j_obj.
-
- .. versionadded:: 1.11.0
- """
-
- def __init__(self, j_obj):
- super().__init__()
- self._j_obj = j_obj
-
- def transform(self, table_env: TableEnvironment, table: Table) -> Table:
- """
- Applies the transformer on the input table, and returns the result table.
-
- :param table_env: the table environment to which the input table is bound.
- :param table: the table to be transformed
- :returns: the transformed table
-
- .. versionadded:: 1.11.0
- """
- self._convert_params_to_java(self._j_obj)
- return Table(self._j_obj.transform(table_env._j_tenv, table._j_table), table_env)
-
-
-class Model(Transformer, metaclass=ABCMeta):
- """
- Abstract class for models that are fitted by estimators.
-
- A model is an ordinary Transformer except how it is created. While ordinary transformers
- are defined by specifying the parameters directly, a model is usually generated by an Estimator
- when Estimator.fit(table_env, table) is invoked.
-
- .. versionadded:: 1.11.0
- """
-
-
-class JavaModel(JavaTransformer, Model):
- """
- Base class for JavaTransformer that wrap Java implementations.
- Subclasses should ensure they have the model Java object available as j_obj.
-
- .. versionadded:: 1.11.0
- """
-
-
-class Estimator(PipelineStage, metaclass=ABCMeta):
- """
- Estimators are PipelineStages responsible for training and generating machine learning models.
-
- The implementations are expected to take an input table as training samples and generate a
- Model which fits these samples.
-
- .. versionadded:: 1.11.0
- """
-
- def fit(self, table_env: TableEnvironment, table: Table) -> Model:
- """
- Train and produce a Model which fits the records in the given Table.
-
- :param table_env: the table environment to which the input table is bound.
- :param table: the table with records to train the Model.
- :returns: a model trained to fit on the given Table.
-
- .. versionadded:: 1.11.0
- """
- raise NotImplementedError()
-
-
-class JavaEstimator(Estimator):
- """
- Base class for Estimator that wrap Java implementations.
- Subclasses should ensure they have the estimator Java object available as j_obj.
-
- .. versionadded:: 1.11.0
- """
-
- def __init__(self, j_obj):
- super().__init__()
- self._j_obj = j_obj
-
- def fit(self, table_env: TableEnvironment, table: Table) -> JavaModel:
- """
- Train and produce a Model which fits the records in the given Table.
-
- :param table_env: the table environment to which the input table is bound.
- :param table: the table with records to train the Model.
- :returns: a model trained to fit on the given Table.
-
- .. versionadded:: 1.11.0
- """
- self._convert_params_to_java(self._j_obj)
- return JavaModel(self._j_obj.fit(table_env._j_tenv, table._j_table))
-
-
-class Pipeline(Estimator, Model, Transformer):
- """
- A pipeline is a linear workflow which chains Estimators and Transformers to
- execute an algorithm.
-
- A pipeline itself can either act as an Estimator or a Transformer, depending on the stages it
- includes. More specifically:
-
-
- If a Pipeline has an Estimator, one needs to call `Pipeline.fit(TableEnvironment, Table)`
- before use the pipeline as a Transformer. In this case the Pipeline is an Estimator and
- can produce a Pipeline as a `Model`.
-
- If a Pipeline has noEstimator, it is a Transformer and can be applied to a Table directly.
- In this case, `Pipeline#fit(TableEnvironment, Table)` will simply return the pipeline itself.
-
-
- In addition, a pipeline can also be used as a PipelineStage in another pipeline, just like an
- ordinaryEstimator or Transformer as describe above.
-
- .. versionadded:: 1.11.0
- """
-
- def __init__(self, stages=None, pipeline_json=None):
- super().__init__()
- self.stages = []
- self.last_estimator_index = -1
- if stages is not None:
- for stage in stages:
- self.append_stage(stage)
- if pipeline_json is not None:
- self.load_json(pipeline_json)
-
- def need_fit(self):
- return self.last_estimator_index >= 0
-
- @staticmethod
- def _is_stage_need_fit(stage):
- return (isinstance(stage, Pipeline) and stage.need_fit()) or \
- ((not isinstance(stage, Pipeline)) and isinstance(stage, Estimator))
-
- def get_stages(self) -> tuple:
- # make it immutable by changing to tuple
- return tuple(self.stages)
-
- def append_stage(self, stage: PipelineStage) -> 'Pipeline':
- if self._is_stage_need_fit(stage):
- self.last_estimator_index = len(self.stages)
- elif not isinstance(stage, Transformer):
- raise RuntimeError("All PipelineStages should be Estimator or Transformer!")
- self.stages.append(stage)
- return self
-
- def fit(self, t_env: TableEnvironment, input: Table) -> 'Pipeline':
- """
- Train the pipeline to fit on the records in the given Table.
-
- :param t_env: the table environment to which the input table is bound.
- :param input: the table with records to train the Pipeline.
- :returns: a pipeline with same stages as this Pipeline except all Estimators \
- replaced with their corresponding Models.
- """
- transform_stages = []
- for i in range(0, len(self.stages)):
- s = self.stages[i]
- if i <= self.last_estimator_index:
- need_fit = self._is_stage_need_fit(s)
- if need_fit:
- t = s.fit(t_env, input)
- else:
- t = s
- transform_stages.append(t)
- input = t.transform(t_env, input)
- else:
- transform_stages.append(s)
- return Pipeline(transform_stages)
-
- def transform(self, t_env: TableEnvironment, input: Table) -> Table:
- """
- Generate a result table by applying all the stages in this pipeline to
- the input table in order.
-
- :param t_env: the table environment to which the input table is bound.
- :param input: the table to be transformed.
- :returns: a result table with all the stages applied to the input tables in order.
- """
- if self.need_fit():
- raise RuntimeError("Pipeline contains Estimator, need to fit first.")
- for s in self.stages:
- input = s.transform(t_env, input)
- return input
-
- def to_json(self) -> str:
- """
- If all PipelineStages in this Pipeline are Java ones, this method will return a
- Java json string, which can be loaded either from a Python Pipeline or a Java Pipeline,
- otherwise, it returns a Python json string which can only be loaded from a Python Pipeline.
- """
- # if all PipelineStages are Java ones, we use Java toJson() to generate Json string
- # so that the string can also be loaded from Java side.
- if all([type(stage) in [JavaTransformer, JavaEstimator, JavaModel]
- for stage in self.get_stages()]):
- j_pipeline = get_gateway().jvm.org.apache.flink.ml.api.core.Pipeline()
- for stage in self.get_stages():
- stage._convert_params_to_java(stage._j_obj)
- j_pipeline.appendStage(stage._j_obj)
- return j_pipeline.toJson()
- else:
- import jsonpickle
- return str(jsonpickle.encode(self, keys=True))
-
- def load_json(self, json: str) -> None:
- """
- This method can either load from a Java Pipeline json or a Python Pipeline json.
- """
- # noinspection PyBroadException
- try:
- # try to load json with Python method
- import jsonpickle
- pipeline = jsonpickle.decode(json, keys=True)
- for stage in pipeline.get_stages():
- self.append_stage(stage)
- except Exception as outer_e:
- try:
- # if can't load json with Python method, try to load with Java method
- gw = get_gateway()
- j_pipeline = gw.jvm.org.apache.flink.ml.api.core.Pipeline()
- j_pipeline.loadJson(json)
-
- for j_stage in j_pipeline.getStages():
- j_stage_class = j_stage.getClass()
- j_transformer_class = java_class(
- gw.jvm.org.apache.flink.ml.api.core.Transformer)
- j_estimator_class = java_class(gw.jvm.org.apache.flink.ml.api.core.Estimator)
- j_model_class = java_class(gw.jvm.org.apache.flink.ml.api.core.Model)
- if j_transformer_class.isAssignableFrom(j_stage_class):
- self.append_stage(JavaTransformer(j_stage))
- elif j_estimator_class.isAssignableFrom(j_stage_class):
- self.append_stage(JavaEstimator(j_stage))
- elif j_model_class.isAssignableFrom(j_stage_class):
- self.append_stage(JavaModel(j_stage))
- else:
- raise TypeError(
- "Unexpected Java PipelineStage %s. Class should be a %s, "
- "%s or a %s." %
- (j_stage_class.getCanonicalName(),
- j_transformer_class.getCanonicalName(),
- j_estimator_class.getCanonicalName(),
- j_model_class.getCanonicalName()))
- except Exception as inner_e:
- raise RuntimeError(
- "Cannot load the JSON as either a Java Pipeline or a Python Pipeline.\n"
- "Python Pipeline load failed due to: %s.\n"
- "Java Pipeline load failed due to: %s." %
- (outer_e, inner_e))
diff --git a/flink-python/pyflink/ml/api/ml_environment.py b/flink-python/pyflink/ml/api/ml_environment.py
deleted file mode 100644
index 7b29ae9..0000000
--- a/flink-python/pyflink/ml/api/ml_environment.py
+++ /dev/null
@@ -1,92 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.dataset.execution_environment import ExecutionEnvironment
-from pyflink.datastream.stream_execution_environment import StreamExecutionEnvironment
-from pyflink.table.table_environment import BatchTableEnvironment, StreamTableEnvironment
-
-
-class MLEnvironment(object):
- """
- The MLEnvironment stores the necessary context in Flink. Each MLEnvironment
- will be associated with a unique ID. The operations associated with the same
- MLEnvironment ID will share the same Flink job context. Both MLEnvironment
- ID and MLEnvironment can only be retrieved from MLEnvironmentFactory.
-
- .. versionadded:: 1.11.0
- """
-
- def __init__(self, exe_env=None, stream_exe_env=None, batch_tab_env=None, stream_tab_env=None):
- self._exe_env = exe_env
- self._stream_exe_env = stream_exe_env
- self._batch_tab_env = batch_tab_env
- self._stream_tab_env = stream_tab_env
-
- def get_execution_environment(self) -> ExecutionEnvironment:
- """
- Get the ExecutionEnvironment. If the ExecutionEnvironment has not been set,
- it initial the ExecutionEnvironment with default Configuration.
-
- :return: the batch ExecutionEnvironment.
-
- .. versionadded:: 1.11.0
- """
- if self._exe_env is None:
- self._exe_env = ExecutionEnvironment.get_execution_environment()
- return self._exe_env
-
- def get_stream_execution_environment(self) -> StreamExecutionEnvironment:
- """
- Get the StreamExecutionEnvironment. If the StreamExecutionEnvironment has not been
- set, it initial the StreamExecutionEnvironment with default Configuration.
-
- :return: the StreamExecutionEnvironment.
-
- .. versionadded:: 1.11.0
- """
- if self._stream_exe_env is None:
- self._stream_exe_env = StreamExecutionEnvironment.get_execution_environment()
- return self._stream_exe_env
-
- def get_batch_table_environment(self) -> BatchTableEnvironment:
- """
- Get the BatchTableEnvironment. If the BatchTableEnvironment has not been set,
- it initial the BatchTableEnvironment with default Configuration.
-
- :return: the BatchTableEnvironment.
-
- .. versionadded:: 1.11.0
- """
- if self._batch_tab_env is None:
- self._batch_tab_env = BatchTableEnvironment.create(
- ExecutionEnvironment.get_execution_environment())
- return self._batch_tab_env
-
- def get_stream_table_environment(self) -> StreamTableEnvironment:
- """
- Get the StreamTableEnvironment. If the StreamTableEnvironment has not been set,
- it initial the StreamTableEnvironment with default Configuration.
-
- :return: the StreamTableEnvironment.
-
- .. versionadded:: 1.11.0
- """
- if self._stream_tab_env is None:
- self._stream_tab_env = StreamTableEnvironment.create(
- StreamExecutionEnvironment.get_execution_environment())
- return self._stream_tab_env
diff --git a/flink-python/pyflink/ml/api/ml_environment_factory.py b/flink-python/pyflink/ml/api/ml_environment_factory.py
deleted file mode 100644
index 4f40b38..0000000
--- a/flink-python/pyflink/ml/api/ml_environment_factory.py
+++ /dev/null
@@ -1,125 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from typing import Optional
-from pyflink.ml.api.ml_environment import MLEnvironment
-from pyflink.dataset.execution_environment import ExecutionEnvironment
-from pyflink.datastream.stream_execution_environment import StreamExecutionEnvironment
-from pyflink.table.table_environment import BatchTableEnvironment, StreamTableEnvironment
-from pyflink.java_gateway import get_gateway
-import threading
-
-
-class MLEnvironmentFactory:
- """
- Factory to get the MLEnvironment using a MLEnvironmentId.
-
- .. versionadded:: 1.11.0
- """
- _lock = threading.RLock()
- _default_ml_environment_id = 0
- _next_id = 1
- _map = {_default_ml_environment_id: None}
-
- @staticmethod
- def get(ml_env_id: int) -> Optional[MLEnvironment]:
- """
- Get the MLEnvironment using a MLEnvironmentId.
-
- :param ml_env_id: the MLEnvironmentId
- :return: the MLEnvironment
-
- .. versionadded:: 1.11.0
- """
- with MLEnvironmentFactory._lock:
- if ml_env_id == 0:
- return MLEnvironmentFactory.get_default()
- elif ml_env_id not in MLEnvironmentFactory._map:
- raise ValueError(
- "Cannot find MLEnvironment for MLEnvironmentId %s. "
- "Did you get the MLEnvironmentId by calling "
- "get_new_ml_environment_id?" % ml_env_id)
- return MLEnvironmentFactory._map[ml_env_id]
-
- @staticmethod
- def get_default() -> Optional[MLEnvironment]:
- """
- Get the MLEnvironment use the default MLEnvironmentId.
-
- :return: the default MLEnvironment.
-
- .. versionadded:: 1.11.0
- """
- with MLEnvironmentFactory._lock:
- if MLEnvironmentFactory._map[MLEnvironmentFactory._default_ml_environment_id] is None:
- j_ml_env = get_gateway().\
- jvm.org.apache.flink.ml.common.MLEnvironmentFactory.getDefault()
- ml_env = MLEnvironment(
- ExecutionEnvironment(j_ml_env.getExecutionEnvironment()),
- StreamExecutionEnvironment(j_ml_env.getStreamExecutionEnvironment()),
- BatchTableEnvironment(j_ml_env.getBatchTableEnvironment()),
- StreamTableEnvironment(j_ml_env.getStreamTableEnvironment()))
- MLEnvironmentFactory._map[MLEnvironmentFactory._default_ml_environment_id] = ml_env
-
- return MLEnvironmentFactory._map[MLEnvironmentFactory._default_ml_environment_id]
-
- @staticmethod
- def get_new_ml_environment_id() -> int:
- """
- Create a unique MLEnvironment id and register a new MLEnvironment in the factory.
-
- :return: the MLEnvironment id.
-
- .. versionadded:: 1.11.0
- """
- with MLEnvironmentFactory._lock:
- return MLEnvironmentFactory.register_ml_environment(MLEnvironment())
-
- @staticmethod
- def register_ml_environment(ml_environment: MLEnvironment) -> int:
- """
- Register a new MLEnvironment to the factory and return a new MLEnvironment id.
-
- :param ml_environment: the MLEnvironment that will be stored in the factory.
- :return: the MLEnvironment id.
-
- .. versionadded:: 1.11.0
- """
- with MLEnvironmentFactory._lock:
- MLEnvironmentFactory._map[MLEnvironmentFactory._next_id] = ml_environment
- MLEnvironmentFactory._next_id += 1
- return MLEnvironmentFactory._next_id - 1
-
- @staticmethod
- def remove(ml_env_id: int) -> MLEnvironment:
- """
- Remove the MLEnvironment using the MLEnvironmentId.
-
- :param ml_env_id: the id.
- :return: the removed MLEnvironment
-
- .. versionadded:: 1.11.0
- """
- with MLEnvironmentFactory._lock:
- if ml_env_id is None:
- raise ValueError("The environment id cannot be null.")
- # Never remove the default MLEnvironment. Just return the default environment.
- if MLEnvironmentFactory._default_ml_environment_id == ml_env_id:
- return MLEnvironmentFactory.get_default()
- else:
- return MLEnvironmentFactory._map.pop(ml_env_id)
diff --git a/flink-python/pyflink/ml/api/param/__init__.py b/flink-python/pyflink/ml/api/param/__init__.py
deleted file mode 100644
index 871d887..0000000
--- a/flink-python/pyflink/ml/api/param/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.api.param.base import WithParams, Params, ParamInfo, TypeConverters
-
-__all__ = ['WithParams', 'Params', 'ParamInfo', 'TypeConverters']
diff --git a/flink-python/pyflink/ml/api/param/base.py b/flink-python/pyflink/ml/api/param/base.py
deleted file mode 100644
index 6931302..0000000
--- a/flink-python/pyflink/ml/api/param/base.py
+++ /dev/null
@@ -1,350 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-import array
-from typing import TypeVar, Generic, List
-
-V = TypeVar('V')
-
-
-class WithParams(Generic[V]):
- """
- Parameters are widely used in machine learning realm. This class defines a common
- interface to interact with classes with parameters.
- """
-
- def get_params(self) -> 'Params':
- """
- Returns all the parameters.
-
- :return: all the parameters.
- """
- pass
-
- def set(self, info: 'ParamInfo', value: V) -> 'WithParams':
- """
- Set the value of a specific parameter.
-
- :param info: the info of the specific param to set.
- :param value: the value to be set to the specific param.
- :return: the WithParams itself.
- """
- self.get_params().set(info, value)
- return self
-
- def get(self, info: 'ParamInfo') -> V:
- """
- Returns the value of the specific param.
-
- :param info: the info of the specific param, usually with default value.
- :return: the value of the specific param, or default value defined in the \
- ParamInfo if the inner Params doesn't contains this param.
- """
- return self.get_params().get(info)
-
- def _set(self, **kwargs):
- """
- Sets user-supplied params.
- """
- for param, value in kwargs.items():
- p = getattr(self, param)
- if value is not None:
- try:
- value = p.type_converter(value)
- except TypeError as e:
- raise TypeError('Invalid param value given for param "%s". %s' % (p.name, e))
- self.get_params().set(p, value)
- return self
-
-
-class Params(Generic[V]):
- """
- The map-like container class for parameter. This class is provided to unify
- the interaction with parameters.
- """
-
- def __init__(self):
- self._param_map = {}
-
- def set(self, info: 'ParamInfo', value: V) -> 'Params':
- """
- Return the number of params.
-
- :param info: the info of the specific parameter to set.
- :param value: the value to be set to the specific parameter.
- :return: return the current Params.
- """
- self._param_map[info] = value
- return self
-
- def get(self, info: 'ParamInfo') -> V:
- """
- Returns the value of the specific parameter, or default value defined in the
- info if this Params doesn't have a value set for the parameter. An exception
- will be thrown in the following cases because no value could be found for the
- specified parameter.
-
- :param info: the info of the specific parameter to set.
- :return: the value of the specific param, or default value defined in the \
- info if this Params doesn't contain the parameter.
- """
- if info not in self._param_map:
- if not info.is_optional:
- raise ValueError("Missing non-optional parameter %s" % info.name)
- elif not info.has_default_value:
- raise ValueError("Cannot find default value for optional parameter %s" % info.name)
- else:
- return info.default_value
- else:
- return self._param_map[info]
-
- def remove(self, info: 'ParamInfo') -> V:
- """
- Removes the specific parameter from this Params.
-
- :param info: the info of the specific parameter to remove.
- :return: the type of the specific parameter.
- """
- self._param_map.pop(info)
-
- def contains(self, info: 'ParamInfo') -> bool:
- """
- Check whether this params has a value set for the given `info`.
-
- :param info: the info of the specific parameter to check.
- :return: `True` if this params has a value set for the specified `info`, false otherwise.
- """
- return info in self._param_map
-
- def size(self) -> int:
- """
- Return the number of params.
-
- :return: Return the number of params.
- """
- return len(self._param_map)
-
- def clear(self) -> None:
- """
- Removes all of the params. The params will be empty after this call returns.
-
- :return: None.
- """
- self._param_map.clear()
-
- def is_empty(self) -> bool:
- """
- Returns `true` if this params contains no mappings.
-
- :return: `true` if this params contains no mappings.
- """
- return len(self._param_map) == 0
-
- def to_json(self) -> str:
- """
- Returns a json containing all parameters in this Params. The json should be
- human-readable if possible.
-
- :return: a json containing all parameters in this Params.
- """
- import jsonpickle
- return str(jsonpickle.encode(self._param_map, keys=True))
-
- def load_json(self, json: str) -> None:
- """
- Restores the parameters from the given json. The parameters should be exactly
- the same with the one who was serialized to the input json after the restoration.
-
- :param json: the json String to restore from.
- :return: None.
- """
- import jsonpickle
- self._param_map.update(jsonpickle.decode(json, keys=True))
-
- @staticmethod
- def from_json(json) -> 'Params':
- """
- Factory method for constructing params.
-
- :param json: the json string to load.
- :return: the `Params` loaded from the json string.
- """
- ret = Params()
- ret.load_json(json)
- return ret
-
- def merge(self, other_params: 'Params') -> 'Params':
- """
- Merge other params into this.
-
- :param other_params: other params.
- :return: return this Params.
- """
- if other_params is not None:
- self._param_map.update(other_params._param_map)
- return self
-
- def clone(self) -> 'Params':
- """
- Creates and returns a deep clone of this Params.
-
- :return: a clone of this Params.
- """
- new_params = Params()
- new_params._param_map.update(self._param_map)
- return new_params
-
-
-class ParamInfo(object):
- """
- Definition of a parameter, including name, description, type_converter and so on.
- """
-
- def __init__(self, name, description, is_optional=True,
- has_default_value=False, default_value=None,
- type_converter=None):
- self.name = str(name)
- self.description = str(description)
- self.is_optional = is_optional
- self.has_default_value = has_default_value
- self.default_value = default_value
- self.type_converter = TypeConverters.identity if type_converter is None else type_converter
-
- def __str__(self):
- return self.name
-
- def __repr__(self):
- return "Param(name=%r, description=%r)" % (self.name, self.description)
-
- def __hash__(self):
- return hash(str(self.name))
-
- def __eq__(self, other):
- if isinstance(other, ParamInfo):
- return self.name == other.name
- else:
- return False
-
-
-class TypeConverters(object):
- """
- Factory methods for common type conversion functions for `Param.typeConverter`.
- The TypeConverter makes PyFlink ML pipeline support more types of parameters. For example,
- a list could be a list, a range or an array. Validation can also be done in the converters.
- """
-
- @staticmethod
- def _is_numeric(value):
- vtype = type(value)
- return vtype in [int, float] or vtype.__name__ == 'long'
-
- @staticmethod
- def _is_integer(value):
- return TypeConverters._is_numeric(value) and float(value).is_integer()
-
- @staticmethod
- def _can_convert_to_string(value):
- return isinstance(value, str)
-
- @staticmethod
- def identity(value):
- """
- Dummy converter that just returns value.
- """
- return value
-
- @staticmethod
- def to_list(value) -> List:
- """
- Convert a value to a list, if possible.
- """
- if isinstance(value, list):
- return value
- elif type(value) in [tuple, range, array.array]:
- return list(value)
- else:
- raise TypeError("Could not convert %s to list" % value)
-
- @staticmethod
- def to_list_float(value) -> List[float]:
- """
- Convert a value to list of floats, if possible.
- """
- value = TypeConverters.to_list(value)
- if all(map(lambda v: TypeConverters._is_numeric(v), value)):
- return [float(v) for v in value]
- raise TypeError("Could not convert %s to list of floats" % value)
-
- @staticmethod
- def to_list_int(value) -> List[int]:
- """
- Convert a value to list of ints, if possible.
- """
- value = TypeConverters.to_list(value)
- if all(map(lambda v: TypeConverters._is_integer(v), value)):
- return [int(v) for v in value]
- raise TypeError("Could not convert %s to list of ints" % value)
-
- @staticmethod
- def to_list_string(value) -> List[str]:
- """
- Convert a value to list of strings, if possible.
- """
- value = TypeConverters.to_list(value)
- if all(map(lambda v: TypeConverters._can_convert_to_string(v), value)):
- return [TypeConverters.to_string(v) for v in value]
- raise TypeError("Could not convert %s to list of strings" % value)
-
- @staticmethod
- def to_float(value: float) -> float:
- """
- Convert a value to a float, if possible.
- """
- if TypeConverters._is_numeric(value):
- return float(value)
- else:
- raise TypeError("Could not convert %s to float" % value)
-
- @staticmethod
- def to_int(value: int) -> int:
- """
- Convert a value to an int, if possible.
- """
- if TypeConverters._is_integer(value):
- return int(value)
- else:
- raise TypeError("Could not convert %s to int" % value)
-
- @staticmethod
- def to_string(value: str) -> str:
- """
- Convert a value to a string, if possible.
- """
- if isinstance(value, str):
- return value
- else:
- raise TypeError("Could not convert %s to string type" % type(value))
-
- @staticmethod
- def to_boolean(value: bool) -> bool:
- """
- Convert a value to a boolean, if possible.
- """
- if isinstance(value, bool):
- return value
- else:
- raise TypeError("Boolean Param requires value of type bool. Found %s." % type(value))
diff --git a/flink-python/pyflink/ml/lib/__init__.py b/flink-python/pyflink/ml/lib/__init__.py
deleted file mode 100644
index 65b48d4..0000000
--- a/flink-python/pyflink/ml/lib/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
diff --git a/flink-python/pyflink/ml/lib/param/__init__.py b/flink-python/pyflink/ml/lib/param/__init__.py
deleted file mode 100644
index e0a2e2c6..0000000
--- a/flink-python/pyflink/ml/lib/param/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.lib.param.colname import HasSelectedCols, HasOutputCol
-
-__all__ = ["HasSelectedCols", "HasOutputCol"]
diff --git a/flink-python/pyflink/ml/lib/param/colname.py b/flink-python/pyflink/ml/lib/param/colname.py
deleted file mode 100644
index 57decd7..0000000
--- a/flink-python/pyflink/ml/lib/param/colname.py
+++ /dev/null
@@ -1,78 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.api.param import WithParams, ParamInfo, TypeConverters
-
-
-class HasSelectedCols(WithParams):
- """
- An interface for classes with a parameter specifying the name of multiple table columns.
-
- .. versionadded:: 1.11.0
- """
-
- selected_cols = ParamInfo(
- "selectedCols",
- "Names of the columns used for processing",
- is_optional=False,
- type_converter=TypeConverters.to_list_string)
-
- def set_selected_cols(self, v: list) -> 'HasSelectedCols':
- return super().set(self.selected_cols, v)
-
- def get_selected_cols(self) -> list:
- return super().get(self.selected_cols)
-
-
-class HasOutputCol(WithParams):
- """
- An interface for classes with a parameter specifying the name of the output column.
-
- .. versionadded:: 1.11.0
- """
-
- output_col = ParamInfo(
- "outputCol",
- "Name of the output column",
- is_optional=False,
- type_converter=TypeConverters.to_string)
-
- def set_output_col(self, v: str) -> 'HasOutputCol':
- return super().set(self.output_col, v)
-
- def get_output_col(self) -> str:
- return super().get(self.output_col)
-
-
-class HasPredictionCol(WithParams):
- """
- An interface for classes with a parameter specifying the column name of the prediction.
-
- .. versionadded:: 1.11.0
- """
- prediction_col = ParamInfo(
- "predictionCol",
- "Column name of prediction.",
- is_optional=False,
- type_converter=TypeConverters.to_string)
-
- def set_prediction_col(self, v: str) -> 'HasPredictionCol':
- return super().set(self.prediction_col, v)
-
- def get_prediction_col(self) -> str:
- return super().get(self.prediction_col)
diff --git a/flink-python/pyflink/ml/tests/__init__.py b/flink-python/pyflink/ml/tests/__init__.py
deleted file mode 100644
index 65b48d4..0000000
--- a/flink-python/pyflink/ml/tests/__init__.py
+++ /dev/null
@@ -1,17 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
diff --git a/flink-python/pyflink/ml/tests/test_ml_environment.py b/flink-python/pyflink/ml/tests/test_ml_environment.py
deleted file mode 100644
index 6cbf6ca..0000000
--- a/flink-python/pyflink/ml/tests/test_ml_environment.py
+++ /dev/null
@@ -1,55 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.api.ml_environment_factory import MLEnvironment
-from pyflink.dataset import ExecutionEnvironment
-from pyflink.datastream import StreamExecutionEnvironment
-from pyflink.table import BatchTableEnvironment, StreamTableEnvironment
-from pyflink.testing.test_case_utils import PyFlinkTestCase
-
-
-class MLEnvironmentTest(PyFlinkTestCase):
-
- def test_default_constructor(self):
- ml_environment = MLEnvironment()
- self.assertIsNotNone(ml_environment.get_execution_environment())
- self.assertIsNotNone(ml_environment.get_stream_execution_environment())
- self.assertIsNotNone(ml_environment.get_batch_table_environment())
- self.assertIsNotNone(ml_environment.get_stream_table_environment())
-
- def test_construct_with_batch_env(self):
- execution_environment = ExecutionEnvironment.get_execution_environment()
- batch_table_environment = BatchTableEnvironment.create(execution_environment)
-
- ml_environment = MLEnvironment(
- exe_env=execution_environment,
- batch_tab_env=batch_table_environment)
- self.assertEqual(ml_environment.get_execution_environment(), execution_environment)
- self.assertEqual(ml_environment.get_batch_table_environment(), batch_table_environment)
-
- def test_construct_with_stream_env(self):
- stream_execution_environment = StreamExecutionEnvironment.get_execution_environment()
- stream_table_environment = StreamTableEnvironment.create(stream_execution_environment)
-
- ml_environment = MLEnvironment(
- stream_exe_env=stream_execution_environment,
- stream_tab_env=stream_table_environment)
- self.assertEqual(
- ml_environment.get_stream_execution_environment(),
- stream_execution_environment)
- self.assertEqual(ml_environment.get_stream_table_environment(), stream_table_environment)
diff --git a/flink-python/pyflink/ml/tests/test_ml_environment_factory.py b/flink-python/pyflink/ml/tests/test_ml_environment_factory.py
deleted file mode 100644
index 7f381d1..0000000
--- a/flink-python/pyflink/ml/tests/test_ml_environment_factory.py
+++ /dev/null
@@ -1,48 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.api.ml_environment_factory import MLEnvironmentFactory, MLEnvironment
-from pyflink.testing.test_case_utils import MLTestCase
-
-
-class MLEnvironmentFactoryTest(MLTestCase):
-
- def test_get_default(self):
- ml_env1 = MLEnvironmentFactory.get_default()
- ml_env2 = MLEnvironmentFactory.get_default()
- self.assertEqual(ml_env1, ml_env2)
-
- def test_register_and_get_ml_environment(self):
- ml_environment = MLEnvironment()
- # test register
- id = MLEnvironmentFactory.register_ml_environment(ml_environment)
- # test get
- ml_environment_2 = MLEnvironmentFactory.get(id)
- self.assertEqual(ml_environment, ml_environment_2)
-
- def test_get_new_ml_environment_id_and_remove(self):
- # test get_new_ml_environment_id
- id = MLEnvironmentFactory.get_new_ml_environment_id()
- ml_environment = MLEnvironmentFactory.get(id)
- # test remove
- ml_environment_2 = MLEnvironmentFactory.remove(id)
- self.assertEqual(ml_environment, ml_environment_2)
- # test remove default
- self.assertEqual(
- MLEnvironmentFactory.remove(0),
- MLEnvironmentFactory.get_default())
diff --git a/flink-python/pyflink/ml/tests/test_params.py b/flink-python/pyflink/ml/tests/test_params.py
deleted file mode 100644
index c4818c8..0000000
--- a/flink-python/pyflink/ml/tests/test_params.py
+++ /dev/null
@@ -1,187 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-import array
-
-from pyflink import keyword
-from pyflink.ml.api.param import ParamInfo, TypeConverters, Params
-from pyflink.ml.lib.param.colname import HasSelectedCols, HasOutputCol
-from pyflink.testing.test_case_utils import PyFlinkTestCase
-
-
-class ParamsTest(PyFlinkTestCase):
-
- def test_default_behavior(self):
- params = Params()
-
- not_optinal = ParamInfo("a", "", is_optional=False)
- with self.assertRaises(ValueError):
- params.get(not_optinal)
-
- # get optional without default param
- optional_without_default = ParamInfo("a", "")
- with self.assertRaises(ValueError):
- params.get(optional_without_default)
-
- def test_get_optional_param(self):
- param_info = ParamInfo(
- "key",
- "",
- has_default_value=True,
- default_value=None,
- type_converter=TypeConverters.to_string)
-
- params = Params()
- self.assertIsNone(params.get(param_info))
-
- val = "3"
- params.set(param_info, val)
- self.assertEqual(val, params.get(param_info))
-
- params.set(param_info, None)
- self.assertIsNone(params.get(param_info))
-
- def test_remove_contains_size_clear_is_empty(self):
- param_info = ParamInfo(
- "key",
- "",
- has_default_value=True,
- default_value=None,
- type_converter=TypeConverters.to_string)
-
- params = Params()
- self.assertEqual(params.size(), 0)
- self.assertTrue(params.is_empty())
-
- val = "3"
- params.set(param_info, val)
- self.assertEqual(params.size(), 1)
- self.assertFalse(params.is_empty())
-
- params_json = params.to_json()
- params_new = Params.from_json(params_json)
- self.assertEqual(params.get(param_info), val)
- self.assertEqual(params_new.get(param_info), val)
-
- params.clear()
- self.assertEqual(params.size(), 0)
- self.assertTrue(params.is_empty())
-
- def test_to_from_json(self):
- import jsonpickle
-
- param_info = ParamInfo(
- "key",
- "",
- has_default_value=True,
- default_value=None,
- type_converter=TypeConverters.to_string)
-
- param_info_new = jsonpickle.decode(jsonpickle.encode(param_info))
- self.assertEqual(param_info_new, param_info)
-
- params = Params()
- val = "3"
- params.set(param_info, val)
- params_new = Params.from_json(params.to_json())
- self.assertEqual(params_new.get(param_info), val)
-
-
-class ParamTypeConversionTests(PyFlinkTestCase):
- """
- Test that param type conversion happens.
- """
-
- def test_list(self):
- l = [0, 1]
- for lst_like in [l, range(2), tuple(l), array.array('l', l)]:
- converted = TypeConverters.to_list(lst_like)
- self.assertEqual(type(converted), list)
- self.assertListEqual(converted, l)
-
- def test_list_float_or_list_int(self):
- l = [0, 1]
- for lst_like in [l, range(2), tuple(l), array.array('l', l)]:
- converted1 = TypeConverters.to_list_float(lst_like)
- converted2 = TypeConverters.to_list_int(lst_like)
- self.assertEqual(type(converted1), list)
- self.assertEqual(type(converted2), list)
- self.assertListEqual(converted1, l)
- self.assertListEqual(converted2, l)
-
- def test_list_string(self):
- l = ["aa", "bb"]
- for lst_like in [l, tuple(l)]:
- converted = TypeConverters.to_list_string(lst_like)
- self.assertEqual(type(converted), list)
- self.assertListEqual(converted, l)
-
- def test_float(self):
- data = 1.45
- converted = TypeConverters.to_float(data)
- self.assertEqual(type(converted), float)
- self.assertEqual(converted, data)
-
- def test_int(self):
- data = 1234567890
- converted = TypeConverters.to_int(data)
- self.assertEqual(type(converted), int)
- self.assertEqual(converted, data)
-
- def test_string(self):
- data = "1234567890"
- converted = TypeConverters.to_string(data)
- self.assertEqual(type(converted), str)
- self.assertEqual(converted, data)
-
- def test_boolean(self):
- data = True
- converted = TypeConverters.to_boolean(data)
- self.assertEqual(type(converted), bool)
- self.assertEqual(converted, data)
-
-
-class MockVectorAssembler(HasSelectedCols, HasOutputCol):
-
- @keyword
- def __init__(self, *, selected_cols=None, output_col=None):
- self._params = Params()
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def get_params(self):
- return self._params
-
-
-class TestWithParams(PyFlinkTestCase):
-
- def test_set_params_with_keyword_arguments(self):
- assembler = MockVectorAssembler(selected_cols=["a", "b"], output_col="features")
- params = assembler.get_params()
- self.assertEqual(params.size(), 2)
- self.assertEqual(assembler.get(HasSelectedCols.selected_cols), ["a", "b"])
- self.assertEqual(assembler.get(HasOutputCol.output_col), "features")
-
- def test_set_params_with_builder_mode(self):
- assembler = MockVectorAssembler()\
- .set_selected_cols(["a", "b"])\
- .set_output_col("features")
- params = assembler.get_params()
- self.assertEqual(params.size(), 2)
- self.assertEqual(assembler.get(HasSelectedCols.selected_cols), ["a", "b"])
- self.assertEqual(assembler.get(HasOutputCol.output_col), "features")
diff --git a/flink-python/pyflink/ml/tests/test_pipeline.py b/flink-python/pyflink/ml/tests/test_pipeline.py
deleted file mode 100644
index b737c44..0000000
--- a/flink-python/pyflink/ml/tests/test_pipeline.py
+++ /dev/null
@@ -1,171 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink.ml.api import JavaTransformer, Transformer, Estimator, Model, \
- Pipeline, JavaEstimator, JavaModel
-from pyflink.ml.api.param.base import WithParams, ParamInfo, TypeConverters
-from pyflink import keyword
-from pyflink.testing.test_case_utils import MLTestCase, PyFlinkTestCase
-
-
-class PipelineTest(PyFlinkTestCase):
-
- @staticmethod
- def describe_pipeline(pipeline):
- res = [stage.get_desc() for stage in pipeline.get_stages()]
- return "_".join(res)
-
- def test_construct_pipeline(self):
- pipeline1 = Pipeline()
- pipeline1.append_stage(MockTransformer(self_desc="a1"))
- pipeline1.append_stage(MockJavaTransformer(self_desc="ja1"))
-
- pipeline2 = Pipeline()
- pipeline2.append_stage(MockTransformer(self_desc="a2"))
- pipeline2.append_stage(MockJavaTransformer(self_desc="ja2"))
-
- pipeline3 = Pipeline(pipeline1.get_stages(), pipeline2.to_json())
- self.assertEqual("a1_ja1_a2_ja2", PipelineTest.describe_pipeline(pipeline3))
-
- def test_pipeline_behavior(self):
- pipeline = Pipeline()
- pipeline.append_stage(MockTransformer(self_desc="a"))
- pipeline.append_stage(MockJavaTransformer(self_desc="ja"))
- pipeline.append_stage(MockEstimator(self_desc="b"))
- pipeline.append_stage(MockJavaEstimator(self_desc="jb"))
- pipeline.append_stage(MockEstimator(self_desc="c"))
- pipeline.append_stage(MockTransformer(self_desc="d"))
- self.assertEqual("a_ja_b_jb_c_d", PipelineTest.describe_pipeline(pipeline))
-
- pipeline_model = pipeline.fit(None, None)
- self.assertEqual("a_ja_mb_mjb_mc_d", PipelineTest.describe_pipeline(pipeline_model))
-
- def test_pipeline_restore(self):
- pipeline = Pipeline()
- pipeline.append_stage(MockTransformer(self_desc="a"))
- pipeline.append_stage(MockJavaTransformer(self_desc="ja"))
- pipeline.append_stage(MockEstimator(self_desc="b"))
- pipeline.append_stage(MockJavaEstimator(self_desc="jb"))
- pipeline.append_stage(MockEstimator(self_desc="c"))
- pipeline.append_stage(MockTransformer(self_desc="d"))
-
- pipeline_new = Pipeline()
- pipeline_new.load_json(pipeline.to_json())
- self.assertEqual("a_ja_b_jb_c_d", PipelineTest.describe_pipeline(pipeline_new))
-
- pipeline_model = pipeline_new.fit(None, None)
- self.assertEqual("a_ja_mb_mjb_mc_d", PipelineTest.describe_pipeline(pipeline_model))
-
-
-class ValidationPipelineTest(MLTestCase):
-
- def test_pipeline_from_invalid_json(self):
- invalid_json = '[a:aa]'
-
- # load json
- p = Pipeline()
- with self.assertRaises(RuntimeError) as context:
- p.load_json(invalid_json)
- exception_str = str(context.exception)
-
- # NOTE: only check the general error message since the detailed error message
- # would be different in different environment.
- self.assertTrue(
- 'Cannot load the JSON as either a Java Pipeline or a Python Pipeline.'
- in exception_str)
- self.assertTrue('Python Pipeline load failed due to:' in exception_str)
- self.assertTrue('Java Pipeline load failed due to:' in exception_str)
- self.assertTrue('JsonParseException' in exception_str)
-
-
-class SelfDescribe(WithParams):
- self_desc = ParamInfo("selfDesc", "selfDesc", type_converter=TypeConverters.to_string)
-
- def set_desc(self, v):
- return super().set(self.self_desc, v)
-
- def get_desc(self):
- return super().get(self.self_desc)
-
-
-class MockTransformer(Transformer, SelfDescribe):
- @keyword
- def __init__(self, *, self_desc=None):
- super().__init__()
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def transform(self, table_env, table):
- return table
-
-
-class MockEstimator(Estimator, SelfDescribe):
- @keyword
- def __init__(self, *, self_desc=None):
- super().__init__()
- self._self_desc = self_desc
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def fit(self, table_env, table):
- return MockModel(self_desc="m" + self._self_desc)
-
-
-class MockModel(Model, SelfDescribe):
- @keyword
- def __init__(self, *, self_desc=None):
- super().__init__()
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def transform(self, table_env, table):
- return table
-
-
-class MockJavaTransformer(JavaTransformer, SelfDescribe):
- @keyword
- def __init__(self, *, self_desc=None):
- super().__init__(None)
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def transform(self, table_env, table):
- return table
-
-
-class MockJavaEstimator(JavaEstimator, SelfDescribe):
- @keyword
- def __init__(self, *, self_desc=None):
- super().__init__(None)
- self._self_desc = self_desc
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def fit(self, table_env, table):
- return MockJavaModel(self_desc="m" + self._self_desc)
-
-
-class MockJavaModel(JavaModel, SelfDescribe):
- @keyword
- def __init__(self, *, self_desc=None):
- super().__init__(None)
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def transform(self, table_env, table):
- return table
diff --git a/flink-python/pyflink/ml/tests/test_pipeline_it_case.py b/flink-python/pyflink/ml/tests/test_pipeline_it_case.py
deleted file mode 100644
index ba43d3f..0000000
--- a/flink-python/pyflink/ml/tests/test_pipeline_it_case.py
+++ /dev/null
@@ -1,189 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from pyflink import keyword
-from pyflink.java_gateway import get_gateway
-from pyflink.ml.api import JavaTransformer, Transformer, Estimator, Model, \
- MLEnvironmentFactory, Pipeline
-from pyflink.ml.api.param import WithParams, ParamInfo, TypeConverters
-from pyflink.ml.lib.param.colname import HasSelectedCols, \
- HasPredictionCol, HasOutputCol
-from pyflink.table.types import DataTypes
-from pyflink.testing import source_sink_utils
-from pyflink.testing.test_case_utils import MLTestCase
-
-
-class HasVectorCol(WithParams):
- """
- Trait for parameter vectorColName.
- """
- vector_col = ParamInfo(
- "vectorCol",
- "Name of a vector column",
- is_optional=False,
- type_converter=TypeConverters.to_string)
-
- def set_vector_col(self, v: str) -> 'HasVectorCol':
- return super().set(self.vector_col, v)
-
- def get_vector_col(self) -> str:
- return super().get(self.vector_col)
-
-
-class WrapperTransformer(JavaTransformer, HasSelectedCols):
- """
- A Transformer wrappers Java Transformer.
- """
-
- @keyword
- def __init__(self, *, selected_cols=None):
- _j_obj = get_gateway().jvm.org.apache.flink.ml.pipeline. \
- UserDefinedPipelineStages.SelectColumnTransformer()
- super().__init__(_j_obj)
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
-
-class PythonAddTransformer(Transformer, HasSelectedCols, HasOutputCol):
- """
- A Transformer which is implemented with Python. Output a column
- contains the sum of all columns.
- """
-
- @keyword
- def __init__(self, *, selected_cols=None, output_col=None):
- super().__init__()
- kwargs = self._input_kwargs
- self._set(**kwargs)
-
- def transform(self, table_env, table):
- input_columns = self.get_selected_cols()
- expr = "+".join(input_columns)
- expr = expr + " as " + self.get_output_col()
- return table.add_columns(expr)
-
-
-class PythonEstimator(Estimator, HasVectorCol, HasPredictionCol):
-
- def __init__(self):
- super().__init__()
-
- def fit(self, table_env, table):
- return PythonModel(
- table_env,
- table.select("max(features) as max_sum"),
- self.get_prediction_col())
-
-
-class PythonModel(Model):
-
- def __init__(self, table_env, model_data_table, output_col_name):
- self._model_data_table = model_data_table
- self._output_col_name = output_col_name
- self.max_sum = 0
- self.load_model(table_env)
-
- def load_model(self, table_env):
- """
- Train the model to get the max_sum value which is used to predict data.
- """
- table_sink = source_sink_utils.TestRetractSink(["max_sum"], [DataTypes.BIGINT()])
- table_env.register_table_sink("Model_Results", table_sink)
- self._model_data_table.execute_insert("Model_Results").wait()
- actual = source_sink_utils.results()
- self.max_sum = actual.apply(0)[3:-1]
-
- def transform(self, table_env, table):
- """
- Use max_sum to predict input. Return turn if input value is bigger than max_sum
- """
- return table \
- .add_columns("features > {} as {}".format(self.max_sum, self._output_col_name)) \
- .select("{}".format(self._output_col_name))
-
-
-class PythonPipelineTest(MLTestCase):
-
- def test_java_transformer(self):
- t_env = MLEnvironmentFactory().get_default().get_stream_table_environment()
-
- table_sink = source_sink_utils.TestAppendSink(
- ['a', 'b'], [DataTypes.BIGINT(), DataTypes.BIGINT()])
- t_env.register_table_sink("TransformerResults", table_sink)
-
- source_table = t_env.from_elements([(1, 2, 3, 4), (4, 3, 2, 1)], ['a', 'b', 'c', 'd'])
- transformer = WrapperTransformer(selected_cols=["a", "b"])
- transformer.transform(t_env, source_table).execute_insert("TransformerResults").wait()
- actual = source_sink_utils.results()
- self.assert_equals(actual, ["+I[1, 2]", "+I[4, 3]"])
-
- def test_pipeline(self):
- t_env = MLEnvironmentFactory().get_default().get_stream_table_environment()
- train_table = t_env.from_elements(
- [(1, 2), (1, 4), (1, 0), (10, 2), (10, 4), (10, 0)], ['a', 'b'])
- serving_table = t_env.from_elements([(0, 0), (12, 3)], ['a', 'b'])
-
- table_sink = source_sink_utils.TestAppendSink(
- ['predict_result'],
- [DataTypes.BOOLEAN()])
- t_env.register_table_sink("PredictResults", table_sink)
-
- # transformer, output features column which is the sum of a and b.
- transformer = PythonAddTransformer(selected_cols=["a", "b"], output_col="features")
-
- # estimator
- estimator = PythonEstimator() \
- .set_vector_col("features") \
- .set_prediction_col("predict_result")
-
- # pipeline
- pipeline = Pipeline().append_stage(transformer).append_stage(estimator)
- pipeline.fit(t_env, train_table).transform(t_env, serving_table) \
- .execute_insert('PredictResults').wait()
-
- actual = source_sink_utils.results()
- # the first input is false since 0 + 0 is smaller than the max_sum 14.
- # the second input is true since 12 + 3 is bigger than the max_sum 14.
- self.assert_equals(actual, ["+I[false]", "+I[true]"])
-
- def test_pipeline_from_and_to_java_json(self):
- # json generated from Java api
- java_json = '[{"stageClassName":"org.apache.flink.ml.pipeline.' \
- 'UserDefinedPipelineStages$SelectColumnTransformer",' \
- '"stageJson":"{\\"selectedCols\\":\\"[\\\\\\"a\\\\\\",' \
- '\\\\\\"b\\\\\\"]\\"}"}]'
-
- # load json
- p = Pipeline()
- p.load_json(java_json)
- python_json = p.to_json()
-
- t_env = MLEnvironmentFactory().get_default().get_stream_table_environment()
-
- table_sink = source_sink_utils.TestAppendSink(
- ['a', 'b'], [DataTypes.BIGINT(), DataTypes.BIGINT()])
- t_env.register_table_sink("TestJsonResults", table_sink)
-
- source_table = t_env.from_elements([(1, 2, 3, 4), (4, 3, 2, 1)], ['a', 'b', 'c', 'd'])
- transformer = p.get_stages()[0]
- transformer.transform(t_env, source_table).execute_insert("TestJsonResults").wait()
-
- actual = source_sink_utils.results()
-
- self.assert_equals(actual, ["+I[1, 2]", "+I[4, 3]"])
- self.assertEqual(python_json, java_json)
diff --git a/flink-python/pyflink/ml/tests/test_pipeline_stage.py b/flink-python/pyflink/ml/tests/test_pipeline_stage.py
deleted file mode 100644
index 5916a35..0000000
--- a/flink-python/pyflink/ml/tests/test_pipeline_stage.py
+++ /dev/null
@@ -1,90 +0,0 @@
-################################################################################
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-################################################################################
-
-from abc import ABC, abstractmethod
-from pyflink.ml.api import Transformer, Estimator, MLEnvironmentFactory
-
-from pyflink.testing.test_case_utils import PyFlinkTestCase
-
-
-class PipelineStageTestBase(ABC):
- """
- The base class for testing the base implementation of pipeline stages, i.e. Estimators
- and Transformers. This class is package private because we do not expect extension outside
- of the package.
- """
-
- @abstractmethod
- def create_pipeline_stage(self):
- pass
-
-
-class TransformerBaseTest(PipelineStageTestBase, PyFlinkTestCase):
- """
- Test for TransformerBase.
- """
- def test_fit_table(self):
- id = MLEnvironmentFactory.get_new_ml_environment_id()
- env = MLEnvironmentFactory.get(id)
- table = env.get_stream_table_environment().from_elements([(1, 2, 3)])
- transformer = self.create_pipeline_stage()
- transformer.transform(env.get_stream_table_environment(), table)
- self.assertTrue(transformer.transformed)
-
- def create_pipeline_stage(self):
- return self.FakeTransFormer()
-
- class FakeTransFormer(Transformer):
- """
- This fake transformer simply record which transform method is invoked.
- """
-
- def __init__(self):
- self.transformed = False
-
- def transform(self, table_env, table):
- self.transformed = True
- return table
-
-
-class EstimatorBaseTest(PipelineStageTestBase, PyFlinkTestCase):
- """
- Test for EstimatorBase.
- """
- def test_fit_table(self):
- id = MLEnvironmentFactory.get_new_ml_environment_id()
- env = MLEnvironmentFactory.get(id)
- table = env.get_stream_table_environment().from_elements([(1, 2, 3)])
- estimator = self.create_pipeline_stage()
- estimator.fit(env.get_stream_table_environment(), table)
- self.assertTrue(estimator.fitted)
-
- def create_pipeline_stage(self):
- return self.FakeEstimator()
-
- class FakeEstimator(Estimator):
- """
- This fake estimator simply record which fit method is invoked.
- """
-
- def __init__(self):
- self.fitted = False
-
- def fit(self, table_env, table):
- self.fitted = True
- return None