You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by sh...@apache.org on 2019/07/05 08:00:49 UTC
[flink] branch master updated: [FLINK-12597][ml] Remove the legacy
flink-ml and flink-ml-uber
This is an automated email from the ASF dual-hosted git repository.
shaoxuan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 2b2a83d [FLINK-12597][ml] Remove the legacy flink-ml and flink-ml-uber
2b2a83d is described below
commit 2b2a83df56242aa90ee731f25d17b050b75df0f3
Author: Gen Luo <lu...@gmail.com>
AuthorDate: Thu Jul 4 00:10:10 2019 +0800
[FLINK-12597][ml] Remove the legacy flink-ml and flink-ml-uber
Remove the legacy flink-libraries/flink-ml and flink-libraries/flink-ml-uber
This closes #8526
---
flink-dist/pom.xml | 7 -
flink-dist/src/main/assemblies/opt.xml | 8 -
flink-libraries/flink-ml-uber/pom.xml | 88 --
.../src/main/resources/META-INF/NOTICE | 30 -
.../META-INF/licenses/LICENSE.arpack_combined_all | 8 -
.../main/resources/META-INF/licenses/LICENSE.core | 49 -
.../META-INF/licenses/LICENSE.jtransforms | 480 ---------
.../resources/META-INF/licenses/LICENSE.machinist | 19 -
.../main/resources/META-INF/licenses/LICENSE.spire | 19 -
flink-libraries/flink-ml/README.md | 22 -
flink-libraries/flink-ml/pom.xml | 201 ----
.../main/scala/org/apache/flink/ml/MLUtils.scala | 126 ---
.../org/apache/flink/ml/classification/SVM.scala | 552 ----------
.../scala/org/apache/flink/ml/common/Block.scala | 29 -
.../org/apache/flink/ml/common/FlinkMLTools.scala | 424 --------
.../org/apache/flink/ml/common/LabeledVector.scala | 42 -
.../org/apache/flink/ml/common/ParameterMap.scala | 121 ---
.../org/apache/flink/ml/common/WeightVector.scala | 32 -
.../apache/flink/ml/common/WithParameters.scala | 26 -
.../main/scala/org/apache/flink/ml/math/BLAS.scala | 291 ------
.../scala/org/apache/flink/ml/math/Breeze.scala | 88 --
.../flink/ml/math/BreezeVectorConverter.scala | 34 -
.../org/apache/flink/ml/math/DenseMatrix.scala | 191 ----
.../org/apache/flink/ml/math/DenseVector.scala | 187 ----
.../scala/org/apache/flink/ml/math/Matrix.scala | 69 --
.../org/apache/flink/ml/math/SparseMatrix.scala | 267 -----
.../org/apache/flink/ml/math/SparseVector.scala | 283 ------
.../scala/org/apache/flink/ml/math/Vector.scala | 103 --
.../org/apache/flink/ml/math/VectorBuilder.scala | 57 --
.../ml/math/distributed/DistributedMatrix.scala | 39 -
.../ml/math/distributed/DistributedRowMatrix.scala | 172 ----
.../scala/org/apache/flink/ml/math/package.scala | 110 --
.../distances/ChebyshevDistanceMetric.scala | 37 -
.../metrics/distances/CosineDistanceMetric.scala | 45 -
.../ml/metrics/distances/DistanceMetric.scala | 37 -
.../distances/EuclideanDistanceMetric.scala | 41 -
.../distances/ManhattanDistanceMetric.scala | 37 -
.../distances/MinkowskiDistanceMetric.scala | 41 -
.../distances/SquaredEuclideanDistanceMetric.scala | 37 -
.../metrics/distances/TanimotoDistanceMetric.scala | 40 -
.../main/scala/org/apache/flink/ml/nn/KNN.scala | 361 -------
.../scala/org/apache/flink/ml/nn/QuadTree.scala | 323 ------
.../flink/ml/optimization/GradientDescent.scala | 297 ------
.../flink/ml/optimization/LossFunction.scala | 96 --
.../ml/optimization/PartialLossFunction.scala | 154 ---
.../flink/ml/optimization/PredictionFunction.scala | 40 -
.../ml/optimization/RegularizationPenalty.scala | 219 ----
.../org/apache/flink/ml/optimization/Solver.scala | 233 -----
.../ml/outlier/StochasticOutlierSelection.scala | 383 -------
.../main/scala/org/apache/flink/ml/package.scala | 119 ---
.../flink/ml/pipeline/ChainedPredictor.scala | 139 ---
.../flink/ml/pipeline/ChainedTransformer.scala | 110 --
.../org/apache/flink/ml/pipeline/Estimator.scala | 181 ----
.../org/apache/flink/ml/pipeline/Predictor.scala | 258 -----
.../org/apache/flink/ml/pipeline/Transformer.scala | 164 ---
.../flink/ml/preprocessing/MinMaxScaler.scala | 265 -----
.../ml/preprocessing/PolynomialFeatures.scala | 209 ----
.../apache/flink/ml/preprocessing/Splitter.scala | 210 ----
.../flink/ml/preprocessing/StandardScaler.scala | 302 ------
.../org/apache/flink/ml/recommendation/ALS.scala | 1060 --------------------
.../ml/regression/MultipleLinearRegression.scala | 234 -----
.../src/test/resources/log4j-test.properties | 38 -
.../flink-ml/src/test/resources/logback-test.xml | 39 -
.../scala/org/apache/flink/ml/MLUtilsSuite.scala | 108 --
.../flink/ml/classification/Classification.scala | 133 ---
.../flink/ml/classification/SVMITSuite.scala | 104 --
.../apache/flink/ml/common/FlinkMLToolsSuite.scala | 60 --
.../org/apache/flink/ml/math/BreezeMathSuite.scala | 98 --
.../apache/flink/ml/math/DenseMatrixSuite.scala | 86 --
.../apache/flink/ml/math/DenseVectorSuite.scala | 176 ----
.../apache/flink/ml/math/SparseMatrixSuite.scala | 134 ---
.../apache/flink/ml/math/SparseVectorSuite.scala | 227 -----
.../distributed/DistributedRowMatrixSuite.scala | 104 --
.../ml/metrics/distances/DistanceMetricSuite.scala | 95 --
.../scala/org/apache/flink/ml/nn/KNNITSuite.scala | 117 ---
.../org/apache/flink/ml/nn/QuadTreeSuite.scala | 93 --
.../ml/optimization/GradientDescentITSuite.scala | 278 -----
.../flink/ml/optimization/LossFunctionTest.scala | 102 --
.../optimization/PredictionFunctionITSuite.scala | 62 --
.../optimization/RegularizationPenaltyTest.scala | 64 --
.../StochasticOutlierSelectionITSuite.scala | 240 -----
.../apache/flink/ml/pipeline/PipelineITSuite.scala | 211 ----
.../ml/preprocessing/MinMaxScalerITSuite.scala | 243 -----
.../preprocessing/PolynomialFeaturesITSuite.scala | 124 ---
.../flink/ml/preprocessing/SplitterITSuite.scala | 97 --
.../ml/preprocessing/StandardScalerITSuite.scala | 166 ---
.../flink/ml/recommendation/ALSITSuite.scala | 116 ---
.../flink/ml/recommendation/Recommendation.scala | 153 ---
.../MultipleLinearRegressionITSuite.scala | 159 ---
.../flink/ml/regression/RegressionData.scala | 206 ----
.../org/apache/flink/ml/util/FlinkTestBase.scala | 73 --
flink-libraries/pom.xml | 2 -
92 files changed, 13754 deletions(-)
diff --git a/flink-dist/pom.xml b/flink-dist/pom.xml
index 771c39c..ca59ba2 100644
--- a/flink-dist/pom.xml
+++ b/flink-dist/pom.xml
@@ -294,13 +294,6 @@ under the License.
<dependency>
<groupId>org.apache.flink</groupId>
- <artifactId>flink-ml-uber_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
<artifactId>flink-table-uber_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<scope>provided</scope>
diff --git a/flink-dist/src/main/assemblies/opt.xml b/flink-dist/src/main/assemblies/opt.xml
index fb58617..ac8bd85 100644
--- a/flink-dist/src/main/assemblies/opt.xml
+++ b/flink-dist/src/main/assemblies/opt.xml
@@ -75,14 +75,6 @@
<fileMode>0644</fileMode>
</file>
- <!-- ML -->
- <file>
- <source>../flink-libraries/flink-ml-uber/target/flink-ml-uber_${scala.binary.version}-${project.version}.jar</source>
- <outputDirectory>opt/</outputDirectory>
- <destName>flink-ml_${scala.binary.version}-${project.version}.jar</destName>
- <fileMode>0644</fileMode>
- </file>
-
<!-- Metrics -->
<file>
<source>../flink-metrics/flink-metrics-graphite/target/flink-metrics-graphite-${project.version}.jar</source>
diff --git a/flink-libraries/flink-ml-uber/pom.xml b/flink-libraries/flink-ml-uber/pom.xml
deleted file mode 100644
index 6931e25..0000000
--- a/flink-libraries/flink-ml-uber/pom.xml
+++ /dev/null
@@ -1,88 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-libraries</artifactId>
- <version>1.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-ml-uber_${scala.binary.version}</artifactId>
- <name>flink-ml-uber</name>
- <description>Utility module for packaging a fat flink-ml jar.</description>
-
- <packaging>jar</packaging>
-
- <dependencies>
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-ml_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
- </dependencies>
-
- <build>
- <plugins>
- <plugin>
- <!-- this is an intermediate build artifact and should not be deployed -->
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-deploy-plugin</artifactId>
- <configuration>
- <skip>true</skip>
- </configuration>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <id>shade-flink</id>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <artifactSet>
- <includes>
- <include>org.apache.flink:flink-ml_*</include>
- <include>com.chuusai:shapeless_*</include>
- <include>net.sf.opencsv:opencsv</include>
- <include>org.scalanlp:breeze_*</include>
- <include>org.scalanlp:breeze-macros_*</include>
- <include>org.typelevel:macro-compat_*</include>
- <include>com.github.fommil.netlib:core</include>
- <include>net.sourceforge.f2j:arpack_combined_all</include>
- <include>org.spire-math:spire_*</include>
- <include>org.spire-math:spire-macros_*</include>
- <include>org.typelevel:machinist_*</include>
- <include>com.github.rwl:jtransforms</include>
- </includes>
- </artifactSet>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/NOTICE b/flink-libraries/flink-ml-uber/src/main/resources/META-INF/NOTICE
deleted file mode 100644
index cbb7d14..0000000
--- a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/NOTICE
+++ /dev/null
@@ -1,30 +0,0 @@
-flink-ml-uber
-Copyright 2014-2019 The Apache Software Foundation
-
-This product includes software developed at
-The Apache Software Foundation (http://www.apache.org/).
-
-This project bundles the following dependencies under the Apache Software License 2.0 (http://www.apache.org/licenses/LICENSE-2.0.txt)
-
-- com.chuusai:shapeless_2.11:2.3.2
-- net.sf.opencsv:opencsv:2.3
-- org.scalanlp:breeze_2.11:0.13
-- org.scalanlp:breeze-macros_2.11:0.13
-- org.typelevel:macro-compat_2.11:1.1.1
-
-This project bundles the following dependencies under the BSD license.
-See bundled license files for details.
-
-- com.github.fommil.netlib:core:1.1.2
-- net.sourceforge.f2j:arpack_combined_all:0.1
-
-This project bundles the following dependencies under the MIT/X11 license.
-See bundled license files for details.
-
-- org.spire-math:spire_2.11:0.13.0
-- org.spire-math:spire-macros_2.11:0.13.0
-- org.typelevel:machinist_2.11:0.6.1
-
-This project bundles the following dependencies under the Mozilla Public license.
-
-- com.github.rwl:jtransforms:2.4.0 (https://sourceforge.net/projects/jtransforms/)
diff --git a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.arpack_combined_all b/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.arpack_combined_all
deleted file mode 100644
index 6a6cdbb..0000000
--- a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.arpack_combined_all
+++ /dev/null
@@ -1,8 +0,0 @@
-Copyright © 2018 The University of Tennessee. All rights reserved.
-
-Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
-· Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
-· Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer listed in this license in the documentation and/or other materials provided with the distribution.
-· Neither the name of the copyright holders nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
-
-This software is provided by the copyright holders and contributors "as is" and any express or implied warranties, including, but not limited to, the implied warranties of merchantability and fitness for a particular purpose are disclaimed. in no event shall the copyright owner or contributors be liable for any direct, indirect, incidental, special, exemplary, or consequential damages (including, but not limited to, procurement of substitute goods or services; loss of use, data, or profi [...]
diff --git a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.core b/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.core
deleted file mode 100644
index b7d2849..0000000
--- a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.core
+++ /dev/null
@@ -1,49 +0,0 @@
-Copyright (c) 2013 Samuel Halliday
-Copyright (c) 1992-2011 The University of Tennessee and The University
- of Tennessee Research Foundation. All rights
- reserved.
-Copyright (c) 2000-2011 The University of California Berkeley. All
- rights reserved.
-Copyright (c) 2006-2011 The University of Colorado Denver. All rights
- reserved.
-
-$COPYRIGHT$
-
-Additional copyrights may follow
-
-$HEADER$
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are
-met:
-
-- Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
-- Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer listed
- in this license in the documentation and/or other materials
- provided with the distribution.
-
-- Neither the name of the copyright holders nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
-The copyright holders provide no reassurances that the source code
-provided does not infringe any patent, copyright, or any other
-intellectual property rights of third parties. The copyright holders
-disclaim any liability to any recipient for claims brought against
-recipient by any third party for infringement of that parties
-intellectual property rights.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
-"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
-LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
-A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
-OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
-SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
-LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
-DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
-THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
-OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
diff --git a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.jtransforms b/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.jtransforms
deleted file mode 100644
index 89169ec..0000000
--- a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.jtransforms
+++ /dev/null
@@ -1,480 +0,0 @@
- MOZILLA PUBLIC LICENSE
- Version 1.1
-
- ---------------
-
-1. Definitions.
-
- 1.0.1. "Commercial Use" means distribution or otherwise making the
- Covered Code available to a third party.
-
- 1.1. "Contributor" means each entity that creates or contributes to
- the creation of Modifications.
-
- 1.2. "Contributor Version" means the combination of the Original
- Code, prior Modifications used by a Contributor, and the Modifications
- made by that particular Contributor.
-
- 1.3. "Covered Code" means the Original Code or Modifications or the
- combination of the Original Code and Modifications, in each case
- including portions thereof.
-
- 1.4. "Electronic Distribution Mechanism" means a mechanism generally
- accepted in the software development community for the electronic
- transfer of data.
-
- 1.5. "Executable" means Covered Code in any form other than Source
- Code.
-
- 1.6. "Initial Developer" means the individual or entity identified
- as the Initial Developer in the Source Code notice required by Exhibit
- A.
-
- 1.7. "Larger Work" means a work which combines Covered Code or
- portions thereof with code not governed by the terms of this License.
-
- 1.8. "License" means this document.
-
- 1.8.1. "Licensable" means having the right to grant, to the maximum
- extent possible, whether at the time of the initial grant or
- subsequently acquired, any and all of the rights conveyed herein.
-
- 1.9. "Modifications" means any addition to or deletion from the
- substance or structure of either the Original Code or any previous
- Modifications. When Covered Code is released as a series of files, a
- Modification is:
- A. Any addition to or deletion from the contents of a file
- containing Original Code or previous Modifications.
-
- B. Any new file that contains any part of the Original Code or
- previous Modifications.
-
- 1.10. "Original Code" means Source Code of computer software code
- which is described in the Source Code notice required by Exhibit A as
- Original Code, and which, at the time of its release under this
- License is not already Covered Code governed by this License.
-
- 1.10.1. "Patent Claims" means any patent claim(s), now owned or
- hereafter acquired, including without limitation, method, process,
- and apparatus claims, in any patent Licensable by grantor.
-
- 1.11. "Source Code" means the preferred form of the Covered Code for
- making modifications to it, including all modules it contains, plus
- any associated interface definition files, scripts used to control
- compilation and installation of an Executable, or source code
- differential comparisons against either the Original Code or another
- well known, available Covered Code of the Contributor's choice. The
- Source Code can be in a compressed or archival form, provided the
- appropriate decompression or de-archiving software is widely available
- for no charge.
-
- 1.12. "You" (or "Your") means an individual or a legal entity
- exercising rights under, and complying with all of the terms of, this
- License or a future version of this License issued under Section 6.1.
- For legal entities, "You" includes any entity which controls, is
- controlled by, or is under common control with You. For purposes of
- this definition, "control" means (a) the power, direct or indirect,
- to cause the direction or management of such entity, whether by
- contract or otherwise, or (b) ownership of more than fifty percent
- (50%) of the outstanding shares or beneficial ownership of such
- entity.
-
-2. Source Code License.
-
- 2.1. The Initial Developer Grant.
- The Initial Developer hereby grants You a world-wide, royalty-free,
- non-exclusive license, subject to third party intellectual property
- claims:
- (a) under intellectual property rights (other than patent or
- trademark) Licensable by Initial Developer to use, reproduce,
- modify, display, perform, sublicense and distribute the Original
- Code (or portions thereof) with or without Modifications, and/or
- as part of a Larger Work; and
-
- (b) under Patents Claims infringed by the making, using or
- selling of Original Code, to make, have made, use, practice,
- sell, and offer for sale, and/or otherwise dispose of the
- Original Code (or portions thereof).
-
- (c) the licenses granted in this Section 2.1(a) and (b) are
- effective on the date Initial Developer first distributes
- Original Code under the terms of this License.
-
- (d) Notwithstanding Section 2.1(b) above, no patent license is
- granted: 1) for code that You delete from the Original Code; 2)
- separate from the Original Code; or 3) for infringements caused
- by: i) the modification of the Original Code or ii) the
- combination of the Original Code with other software or devices.
-
- 2.2. Contributor Grant.
- Subject to third party intellectual property claims, each Contributor
- hereby grants You a world-wide, royalty-free, non-exclusive license
-
- (a) under intellectual property rights (other than patent or
- trademark) Licensable by Contributor, to use, reproduce, modify,
- display, perform, sublicense and distribute the Modifications
- created by such Contributor (or portions thereof) either on an
- unmodified basis, with other Modifications, as Covered Code
- and/or as part of a Larger Work; and
-
- (b) under Patent Claims infringed by the making, using, or
- selling of Modifications made by that Contributor either alone
- and/or in combination with its Contributor Version (or portions
- of such combination), to make, use, sell, offer for sale, have
- made, and/or otherwise dispose of: 1) Modifications made by that
- Contributor (or portions thereof); and 2) the combination of
- Modifications made by that Contributor with its Contributor
- Version (or portions of such combination).
-
- (c) the licenses granted in Sections 2.2(a) and 2.2(b) are
- effective on the date Contributor first makes Commercial Use of
- the Covered Code.
-
- (d) Notwithstanding Section 2.2(b) above, no patent license is
- granted: 1) for any code that Contributor has deleted from the
- Contributor Version; 2) separate from the Contributor Version;
- 3) for infringements caused by: i) third party modifications of
- Contributor Version or ii) the combination of Modifications made
- by that Contributor with other software (except as part of the
- Contributor Version) or other devices; or 4) under Patent Claims
- infringed by Covered Code in the absence of Modifications made by
- that Contributor.
-
-3. Distribution Obligations.
-
- 3.1. Application of License.
- The Modifications which You create or to which You contribute are
- governed by the terms of this License, including without limitation
- Section 2.2. The Source Code version of Covered Code may be
- distributed only under the terms of this License or a future version
- of this License released under Section 6.1, and You must include a
- copy of this License with every copy of the Source Code You
- distribute. You may not offer or impose any terms on any Source Code
- version that alters or restricts the applicable version of this
- License or the recipients' rights hereunder. However, You may include
- an additional document offering the additional rights described in
- Section 3.5.
-
- 3.2. Availability of Source Code.
- Any Modification which You create or to which You contribute must be
- made available in Source Code form under the terms of this License
- either on the same media as an Executable version or via an accepted
- Electronic Distribution Mechanism to anyone to whom you made an
- Executable version available; and if made available via Electronic
- Distribution Mechanism, must remain available for at least twelve (12)
- months after the date it initially became available, or at least six
- (6) months after a subsequent version of that particular Modification
- has been made available to such recipients. You are responsible for
- ensuring that the Source Code version remains available even if the
- Electronic Distribution Mechanism is maintained by a third party.
-
- 3.3. Description of Modifications.
- You must cause all Covered Code to which You contribute to contain a
- file documenting the changes You made to create that Covered Code and
- the date of any change. You must include a prominent statement that
- the Modification is derived, directly or indirectly, from Original
- Code provided by the Initial Developer and including the name of the
- Initial Developer in (a) the Source Code, and (b) in any notice in an
- Executable version or related documentation in which You describe the
- origin or ownership of the Covered Code.
-
- 3.4. Intellectual Property Matters
- (a) Third Party Claims.
- If Contributor has knowledge that a license under a third party's
- intellectual property rights is required to exercise the rights
- granted by such Contributor under Sections 2.1 or 2.2,
- Contributor must include a text file with the Source Code
- distribution titled "LEGAL" which describes the claim and the
- party making the claim in sufficient detail that a recipient will
- know whom to contact. If Contributor obtains such knowledge after
- the Modification is made available as described in Section 3.2,
- Contributor shall promptly modify the LEGAL file in all copies
- Contributor makes available thereafter and shall take other steps
- (such as notifying appropriate mailing lists or newsgroups)
- reasonably calculated to inform those who received the Covered
- Code that new knowledge has been obtained.
-
- (b) Contributor APIs.
- If Contributor's Modifications include an application programming
- interface and Contributor has knowledge of patent licenses which
- are reasonably necessary to implement that API, Contributor must
- also include this information in the LEGAL file.
-
- (c) Representations.
- Contributor represents that, except as disclosed pursuant to
- Section 3.4(a) above, Contributor believes that Contributor's
- Modifications are Contributor's original creation(s) and/or
- Contributor has sufficient rights to grant the rights conveyed by
- this License.
-
- 3.5. Required Notices.
- You must duplicate the notice in Exhibit A in each file of the Source
- Code. If it is not possible to put such notice in a particular Source
- Code file due to its structure, then You must include such notice in a
- location (such as a relevant directory) where a user would be likely
- to look for such a notice. If You created one or more Modification(s)
- You may add your name as a Contributor to the notice described in
- Exhibit A. You must also duplicate this License in any documentation
- for the Source Code where You describe recipients' rights or ownership
- rights relating to Covered Code. You may choose to offer, and to
- charge a fee for, warranty, support, indemnity or liability
- obligations to one or more recipients of Covered Code. However, You
- may do so only on Your own behalf, and not on behalf of the Initial
- Developer or any Contributor. You must make it absolutely clear than
- any such warranty, support, indemnity or liability obligation is
- offered by You alone, and You hereby agree to indemnify the Initial
- Developer and every Contributor for any liability incurred by the
- Initial Developer or such Contributor as a result of warranty,
- support, indemnity or liability terms You offer.
-
- 3.6. Distribution of Executable Versions.
- You may distribute Covered Code in Executable form only if the
- requirements of Section 3.1-3.5 have been met for that Covered Code,
- and if You include a notice stating that the Source Code version of
- the Covered Code is available under the terms of this License,
- including a description of how and where You have fulfilled the
- obligations of Section 3.2. The notice must be conspicuously included
- in any notice in an Executable version, related documentation or
- collateral in which You describe recipients' rights relating to the
- Covered Code. You may distribute the Executable version of Covered
- Code or ownership rights under a license of Your choice, which may
- contain terms different from this License, provided that You are in
- compliance with the terms of this License and that the license for the
- Executable version does not attempt to limit or alter the recipient's
- rights in the Source Code version from the rights set forth in this
- License. If You distribute the Executable version under a different
- license You must make it absolutely clear that any terms which differ
- from this License are offered by You alone, not by the Initial
- Developer or any Contributor. You hereby agree to indemnify the
- Initial Developer and every Contributor for any liability incurred by
- the Initial Developer or such Contributor as a result of any such
- terms You offer.
-
- 3.7. Larger Works.
- You may create a Larger Work by combining Covered Code with other code
- not governed by the terms of this License and distribute the Larger
- Work as a single product. In such a case, You must make sure the
- requirements of this License are fulfilled for the Covered Code.
-
-4. Inability to Comply Due to Statute or Regulation.
-
- If it is impossible for You to comply with any of the terms of this
- License with respect to some or all of the Covered Code due to
- statute, judicial order, or regulation then You must: (a) comply with
- the terms of this License to the maximum extent possible; and (b)
- describe the limitations and the code they affect. Such description
- must be included in the LEGAL file described in Section 3.4 and must
- be included with all distributions of the Source Code. Except to the
- extent prohibited by statute or regulation, such description must be
- sufficiently detailed for a recipient of ordinary skill to be able to
- understand it.
-
-5. Application of this License.
-
- This License applies to code to which the Initial Developer has
- attached the notice in Exhibit A and to related Covered Code.
-
-6. Versions of the License.
-
- 6.1. New Versions.
- Netscape Communications Corporation ("Netscape") may publish revised
- and/or new versions of the License from time to time. Each version
- will be given a distinguishing version number.
-
- 6.2. Effect of New Versions.
- Once Covered Code has been published under a particular version of the
- License, You may always continue to use it under the terms of that
- version. You may also choose to use such Covered Code under the terms
- of any subsequent version of the License published by Netscape. No one
- other than Netscape has the right to modify the terms applicable to
- Covered Code created under this License.
-
- 6.3. Derivative Works.
- If You create or use a modified version of this License (which you may
- only do in order to apply it to code which is not already Covered Code
- governed by this License), You must (a) rename Your license so that
- the phrases "Mozilla", "MOZILLAPL", "MOZPL", "Netscape",
- "MPL", "NPL" or any confusingly similar phrase do not appear in your
- license (except to note that your license differs from this License)
- and (b) otherwise make it clear that Your version of the license
- contains terms which differ from the Mozilla Public License and
- Netscape Public License. (Filling in the name of the Initial
- Developer, Original Code or Contributor in the notice described in
- Exhibit A shall not of themselves be deemed to be modifications of
- this License.)
-
-7. DISCLAIMER OF WARRANTY.
-
- COVERED CODE IS PROVIDED UNDER THIS LICENSE ON AN "AS IS" BASIS,
- WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING,
- WITHOUT LIMITATION, WARRANTIES THAT THE COVERED CODE IS FREE OF
- DEFECTS, MERCHANTABLE, FIT FOR A PARTICULAR PURPOSE OR NON-INFRINGING.
- THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE COVERED CODE
- IS WITH YOU. SHOULD ANY COVERED CODE PROVE DEFECTIVE IN ANY RESPECT,
- YOU (NOT THE INITIAL DEVELOPER OR ANY OTHER CONTRIBUTOR) ASSUME THE
- COST OF ANY NECESSARY SERVICING, REPAIR OR CORRECTION. THIS DISCLAIMER
- OF WARRANTY CONSTITUTES AN ESSENTIAL PART OF THIS LICENSE. NO USE OF
- ANY COVERED CODE IS AUTHORIZED HEREUNDER EXCEPT UNDER THIS DISCLAIMER.
-
-8. TERMINATION.
-
- 8.1. This License and the rights granted hereunder will terminate
- automatically if You fail to comply with terms herein and fail to cure
- such breach within 30 days of becoming aware of the breach. All
- sublicenses to the Covered Code which are properly granted shall
- survive any termination of this License. Provisions which, by their
- nature, must remain in effect beyond the termination of this License
- shall survive.
-
- 8.2. If You initiate litigation by asserting a patent infringement
- claim (excluding declatory judgment actions) against Initial Developer
- or a Contributor (the Initial Developer or Contributor against whom
- You file such action is referred to as "Participant") alleging that:
-
- (a) such Participant's Contributor Version directly or indirectly
- infringes any patent, then any and all rights granted by such
- Participant to You under Sections 2.1 and/or 2.2 of this License
- shall, upon 60 days notice from Participant terminate prospectively,
- unless if within 60 days after receipt of notice You either: (i)
- agree in writing to pay Participant a mutually agreeable reasonable
- royalty for Your past and future use of Modifications made by such
- Participant, or (ii) withdraw Your litigation claim with respect to
- the Contributor Version against such Participant. If within 60 days
- of notice, a reasonable royalty and payment arrangement are not
- mutually agreed upon in writing by the parties or the litigation claim
- is not withdrawn, the rights granted by Participant to You under
- Sections 2.1 and/or 2.2 automatically terminate at the expiration of
- the 60 day notice period specified above.
-
- (b) any software, hardware, or device, other than such Participant's
- Contributor Version, directly or indirectly infringes any patent, then
- any rights granted to You by such Participant under Sections 2.1(b)
- and 2.2(b) are revoked effective as of the date You first made, used,
- sold, distributed, or had made, Modifications made by that
- Participant.
-
- 8.3. If You assert a patent infringement claim against Participant
- alleging that such Participant's Contributor Version directly or
- indirectly infringes any patent where such claim is resolved (such as
- by license or settlement) prior to the initiation of patent
- infringement litigation, then the reasonable value of the licenses
- granted by such Participant under Sections 2.1 or 2.2 shall be taken
- into account in determining the amount or value of any payment or
- license.
-
- 8.4. In the event of termination under Sections 8.1 or 8.2 above,
- all end user license agreements (excluding distributors and resellers)
- which have been validly granted by You or any distributor hereunder
- prior to termination shall survive termination.
-
-9. LIMITATION OF LIABILITY.
-
- UNDER NO CIRCUMSTANCES AND UNDER NO LEGAL THEORY, WHETHER TORT
- (INCLUDING NEGLIGENCE), CONTRACT, OR OTHERWISE, SHALL YOU, THE INITIAL
- DEVELOPER, ANY OTHER CONTRIBUTOR, OR ANY DISTRIBUTOR OF COVERED CODE,
- OR ANY SUPPLIER OF ANY OF SUCH PARTIES, BE LIABLE TO ANY PERSON FOR
- ANY INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES OF ANY
- CHARACTER INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF GOODWILL,
- WORK STOPPAGE, COMPUTER FAILURE OR MALFUNCTION, OR ANY AND ALL OTHER
- COMMERCIAL DAMAGES OR LOSSES, EVEN IF SUCH PARTY SHALL HAVE BEEN
- INFORMED OF THE POSSIBILITY OF SUCH DAMAGES. THIS LIMITATION OF
- LIABILITY SHALL NOT APPLY TO LIABILITY FOR DEATH OR PERSONAL INJURY
- RESULTING FROM SUCH PARTY'S NEGLIGENCE TO THE EXTENT APPLICABLE LAW
- PROHIBITS SUCH LIMITATION. SOME JURISDICTIONS DO NOT ALLOW THE
- EXCLUSION OR LIMITATION OF INCIDENTAL OR CONSEQUENTIAL DAMAGES, SO
- THIS EXCLUSION AND LIMITATION MAY NOT APPLY TO YOU.
-
-10. U.S. GOVERNMENT END USERS.
-
- The Covered Code is a "commercial item," as that term is defined in
- 48 C.F.R. 2.101 (Oct. 1995), consisting of "commercial computer
- software" and "commercial computer software documentation," as such
- terms are used in 48 C.F.R. 12.212 (Sept. 1995). Consistent with 48
- C.F.R. 12.212 and 48 C.F.R. 227.7202-1 through 227.7202-4 (June 1995),
- all U.S. Government End Users acquire Covered Code with only those
- rights set forth herein.
-
-11. MISCELLANEOUS.
-
- This License represents the complete agreement concerning subject
- matter hereof. If any provision of this License is held to be
- unenforceable, such provision shall be reformed only to the extent
- necessary to make it enforceable. This License shall be governed by
- California law provisions (except to the extent applicable law, if
- any, provides otherwise), excluding its conflict-of-law provisions.
- With respect to disputes in which at least one party is a citizen of,
- or an entity chartered or registered to do business in the United
- States of America, any litigation relating to this License shall be
- subject to the jurisdiction of the Federal Courts of the Northern
- District of California, with venue lying in Santa Clara County,
- California, with the losing party responsible for costs, including
- without limitation, court costs and reasonable attorneys' fees and
- expenses. The application of the United Nations Convention on
- Contracts for the International Sale of Goods is expressly excluded.
- Any law or regulation which provides that the language of a contract
- shall be construed against the drafter shall not apply to this
- License.
-
-12. RESPONSIBILITY FOR CLAIMS.
-
- As between Initial Developer and the Contributors, each party is
- responsible for claims and damages arising, directly or indirectly,
- out of its utilization of rights under this License and You agree to
- work with Initial Developer and Contributors to distribute such
- responsibility on an equitable basis. Nothing herein is intended or
- shall be deemed to constitute any admission of liability.
-
-13. MULTIPLE-LICENSED CODE.
-
- Initial Developer may designate portions of the Covered Code as
- "Multiple-Licensed". "Multiple-Licensed" means that the Initial
- Developer permits you to utilize portions of the Covered Code under
- Your choice of the MPL or the alternative licenses, if any, specified
- by the Initial Developer in the file described in Exhibit A.
-
-EXHIBIT A -Mozilla Public License.
-
- ``The contents of this file are subject to the Mozilla Public License
- Version 1.1 (the "License"); you may not use this file except in
- compliance with the License. You may obtain a copy of the License at
- https://www.mozilla.org/MPL/
-
- Software distributed under the License is distributed on an "AS IS"
- basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the
- License for the specific language governing rights and limitations
- under the License.
-
- The Original Code is ______________________________________.
-
- The Initial Developer of the Original Code is ________________________.
- Portions created by ______________________ are Copyright (C) ______
- _______________________. All Rights Reserved.
-
- Contributor(s): ______________________________________.
-
- Alternatively, the contents of this file may be used under the terms
- of the _____ license (the "[___] License"), in which case the
- provisions of [______] License are applicable instead of those
- above. If you wish to allow use of your version of this file only
- under the terms of the [____] License and not to allow others to use
- your version of this file under the MPL, indicate your decision by
- deleting the provisions above and replace them with the notice and
- other provisions required by the [___] License. If you do not delete
- the provisions above, a recipient may use your version of this file
- under either the MPL or the [___] License."
-
- [NOTE: The text of this Exhibit A may differ slightly from the text of
- the notices in the Source Code files of the Original Code. You should
- use the text of this Exhibit A rather than the text found in the
- Original Code Source Code for Your Modifications.]
-
-The contents of this file are subject to the Mozilla Public License Version 1.1 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.mozilla.org/MPL/
-
-Software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License for the specific language governing rights and limitations under the License.
-
-The Original Code is JTransforms.
-
-The Initial Developer of the Original Code is
-Piotr Wendykier, Emory University.
-Portions created by the Initial Developer are Copyright (C) 2007-2009
-the Initial Developer. All Rights Reserved.
diff --git a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.machinist b/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.machinist
deleted file mode 100644
index c4a7a1d..0000000
--- a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.machinist
+++ /dev/null
@@ -1,19 +0,0 @@
-Copyright (c) 2011-2014 Erik Osheim, Tom Switzer
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.spire b/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.spire
deleted file mode 100644
index 645cba6..0000000
--- a/flink-libraries/flink-ml-uber/src/main/resources/META-INF/licenses/LICENSE.spire
+++ /dev/null
@@ -1,19 +0,0 @@
-Copyright (c) 2011-2012 Erik Osheim, Tom Switzer
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of
-this software and associated documentation files (the "Software"), to deal in
-the Software without restriction, including without limitation the rights to
-use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
-of the Software, and to permit persons to whom the Software is furnished to do
-so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
diff --git a/flink-libraries/flink-ml/README.md b/flink-libraries/flink-ml/README.md
deleted file mode 100644
index 5cabd7c..0000000
--- a/flink-libraries/flink-ml/README.md
+++ /dev/null
@@ -1,22 +0,0 @@
-Flink-ML constitutes the machine learning library of Apache Flink.
-Our vision is to make machine learning easily accessible to a wide audience and yet to achieve extraordinary performance.
-For this purpose, Flink-ML is based on two pillars:
-
-Flink-ML contains implementations of popular ML algorithms which are highly optimized for Apache Flink.
-Theses implementations allow to scale to data sizes which vastly exceed the memory of a single computer.
-Flink-ML currently comprises the following algorithms:
-
-* Classification
-** Soft-margin SVM
-* Regression
-** Multiple linear regression
-* Recommendation
-** Alternating least squares (ALS)
-
-Since most of the work in data analytics is related to post- and pre-processing of data where the performance is not crucial, Flink wants to offer a simple abstraction to do that.
-Linear algebra, as common ground of many ML algorithms, represents such a high-level abstraction.
-Therefore, Flink will support the Mahout DSL as a execution engine and provide tools to neatly integrate the optimized algorithms into a linear algebra program.
-
-Flink-ML has just been recently started.
-As part of Apache Flink, it heavily relies on the active work and contributions of its community and others.
-Thus, if you want to add a new algorithm to the library, then find out [how to contribute]((http://flink.apache.org/how-to-contribute.html)) and open a pull request!
\ No newline at end of file
diff --git a/flink-libraries/flink-ml/pom.xml b/flink-libraries/flink-ml/pom.xml
deleted file mode 100644
index 8e6f55c..0000000
--- a/flink-libraries/flink-ml/pom.xml
+++ /dev/null
@@ -1,201 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one
- ~ or more contributor license agreements. See the NOTICE file
- ~ distributed with this work for additional information
- ~ regarding copyright ownership. The ASF licenses this file
- ~ to you under the Apache License, Version 2.0 (the
- ~ "License"); you may not use this file except in compliance
- ~ with the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
- xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
-
- <modelVersion>4.0.0</modelVersion>
-
- <parent>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-libraries</artifactId>
- <version>1.9-SNAPSHOT</version>
- <relativePath>..</relativePath>
- </parent>
-
- <artifactId>flink-ml_${scala.binary.version}</artifactId>
- <name>flink-ml</name>
-
- <packaging>jar</packaging>
-
- <dependencies>
-
- <!-- core dependencies -->
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-scala_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scalanlp</groupId>
- <artifactId>breeze_${scala.binary.version}</artifactId>
- <version>0.13</version>
- </dependency>
-
- <!-- the dependencies below are already provided in Flink -->
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-library</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-compiler</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.commons</groupId>
- <artifactId>commons-math3</artifactId>
- <scope>provided</scope>
- </dependency>
-
- <!-- test dependencies -->
-
- <dependency>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest_${scala.binary.version}</artifactId>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-clients_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
-
- <dependency>
- <groupId>org.apache.flink</groupId>
- <artifactId>flink-test-utils_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>org.scala-lang.modules</groupId>
- <artifactId>scala-xml_${scala.binary.version}</artifactId>
- <version>1.0.5</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
-
- <profiles>
- <profile>
- <id>windows</id>
- <activation>
- <property>
- <name>windows</name>
- </property>
- <os>
- <family>windows</family>
- </os>
- </activation>
- <properties>
- <suffix.test>"(?<!(IT|Integration))(Test|Suite|Case)"</suffix.test>
- <suffix.it>"(IT|Integration)(Test|Suite|Case)"</suffix.it>
- </properties>
- </profile>
- <profile>
- <id>default</id>
- <activation>
- <activeByDefault>true</activeByDefault>
- <property>
- <name>default</name>
- </property>
- </activation>
- <properties>
- <suffix.test>(?<!(IT|Integration))(Test|Suite|Case)</suffix.test>
- <suffix.it>(IT|Integration)(Test|Suite|Case)</suffix.it>
- </properties>
- </profile>
- </profiles>
-
- <build>
- <plugins>
- <plugin>
- <groupId>net.alchim31.maven</groupId>
- <artifactId>scala-maven-plugin</artifactId>
- <executions>
- <execution>
- <goals>
- <goal>compile</goal>
- <goal>testCompile</goal>
- </goals>
- </execution>
- </executions>
- </plugin>
-
- <plugin>
- <groupId>org.scalatest</groupId>
- <artifactId>scalatest-maven-plugin</artifactId>
- <version>1.0</version>
- <configuration>
- <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory>
- <stdout>W</stdout> <!-- Skip coloring output -->
- </configuration>
- <executions>
- <execution>
- <id>scala-test</id>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <suffixes>${suffix.test}</suffixes>
- <argLine>-Xms256m -Xmx800m -Dlog4j.configuration=${log4j.configuration} -Dlog.dir=${log.dir} -Dmvn.forkNumber=1 -XX:-UseGCOverheadLimit</argLine>
- </configuration>
- </execution>
- <execution>
- <id>integration-test</id>
- <phase>integration-test</phase>
- <goals>
- <goal>test</goal>
- </goals>
- <configuration>
- <suffixes>${suffix.it}</suffixes>
- <argLine>-Xms256m -Xmx800m -Dlog4j.configuration=${log4j.configuration} -Dlog.dir=${log.dir} -Dmvn.forkNumber=1 -XX:-UseGCOverheadLimit</argLine>
- </configuration>
- </execution>
- </executions>
- </plugin>
-
- <!-- Scala Code Style, most of the configuration done via plugin management -->
- <plugin>
- <groupId>org.scalastyle</groupId>
- <artifactId>scalastyle-maven-plugin</artifactId>
- <configuration>
- <configLocation>${project.basedir}/../../tools/maven/scalastyle-config.xml</configLocation>
- </configuration>
- </plugin>
- </plugins>
- </build>
-</project>
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/MLUtils.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/MLUtils.scala
deleted file mode 100644
index 4c4072a..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/MLUtils.scala
+++ /dev/null
@@ -1,126 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml
-
-import org.apache.flink.api.common.functions.{RichFlatMapFunction, RichMapFunction}
-import org.apache.flink.api.java.operators.DataSink
-import org.apache.flink.api.scala._
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.ml.common.LabeledVector
-import org.apache.flink.ml.math.SparseVector
-import org.apache.flink.util.Collector
-
-/** Convenience functions for machine learning tasks
- *
- * This object contains convenience functions for machine learning tasks:
- *
- * - readLibSVM:
- * Reads a libSVM/SVMLight input file and returns a data set of [[LabeledVector]].
- * The file format is specified [http://svmlight.joachims.org/ here].
- *
- * - writeLibSVM:
- * Writes a data set of [[LabeledVector]] in libSVM/SVMLight format to disk. The file format
- * is specified [http://svmlight.joachims.org/ here].
- */
-object MLUtils {
-
- val DIMENSION = "dimension"
-
- /** Reads a file in libSVM/SVMLight format and converts the data into a data set of
- * [[LabeledVector]]. The dimension of the [[LabeledVector]] is determined automatically.
- *
- * Since the libSVM/SVMLight format stores a vector in its sparse form, the [[LabeledVector]]
- * will also be instantiated with a [[SparseVector]].
- *
- * @param env executionEnvironment [[ExecutionEnvironment]]
- * @param filePath Path to the input file
- * @return [[DataSet]] of [[LabeledVector]] containing the information of the libSVM/SVMLight
- * file
- */
- def readLibSVM(env: ExecutionEnvironment, filePath: String): DataSet[LabeledVector] = {
- val labelCOODS = env.readTextFile(filePath).flatMap(
- new RichFlatMapFunction[String, (Double, Array[(Int, Double)])] {
- val splitPattern = "\\s+".r
-
- override def flatMap(
- line: String,
- out: Collector[(Double, Array[(Int, Double)])]
- ): Unit = {
- val commentFreeLine = line.takeWhile(_ != '#').trim
-
- if (commentFreeLine.nonEmpty) {
- val splits = splitPattern.split(commentFreeLine)
- val label = splits.head.toDouble
- val sparseFeatures = splits.tail
- val coos = sparseFeatures.flatMap { str =>
- val pair = str.split(':')
- require(pair.length == 2, "Each feature entry has to have the form <feature>:<value>")
-
- // libSVM index is 1-based, but we expect it to be 0-based
- val index = pair(0).toInt - 1
- val value = pair(1).toDouble
-
- Some((index, value))
- }
-
- out.collect((label, coos))
- }
- }
- })
-
- // Calculate maximum dimension of vectors
- val dimensionDS = labelCOODS.map {
- labelCOO =>
- labelCOO._2.map( _._1 + 1 ).max
- }.reduce(scala.math.max(_, _))
-
- labelCOODS.map{ new RichMapFunction[(Double, Array[(Int, Double)]), LabeledVector] {
- var dimension = 0
-
- override def open(configuration: Configuration): Unit = {
- dimension = getRuntimeContext.getBroadcastVariable(DIMENSION).get(0)
- }
-
- override def map(value: (Double, Array[(Int, Double)])): LabeledVector = {
- new LabeledVector(value._1, SparseVector.fromCOO(dimension, value._2))
- }
- }}.withBroadcastSet(dimensionDS, DIMENSION)
- }
-
- /** Writes a [[DataSet]] of [[LabeledVector]] to a file using the libSVM/SVMLight format.
- *
- * @param filePath Path to output file
- * @param labeledVectors [[DataSet]] of [[LabeledVector]] to write to disk
- * @return
- */
- def writeLibSVM(filePath: String, labeledVectors: DataSet[LabeledVector]): DataSink[String] = {
- val stringRepresentation = labeledVectors.map{
- labeledVector =>
- val vectorStr = labeledVector.vector.
- // remove zero entries
- filter( _._2 != 0).
- map{case (idx, value) => (idx + 1) + ":" + value}.
- mkString(" ")
-
- labeledVector.label + " " + vectorStr
- }
-
- stringRepresentation.writeAsText(filePath)
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala
deleted file mode 100644
index 6d78ef7..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/classification/SVM.scala
+++ /dev/null
@@ -1,552 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.classification
-
-import org.apache.flink.api.common.functions.RichMapFunction
-import org.apache.flink.api.scala._
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.ml.common.FlinkMLTools.ModuloKeyPartitioner
-import org.apache.flink.ml.common._
-import org.apache.flink.ml.math.Breeze._
-import org.apache.flink.ml.math.{DenseVector, Vector}
-import org.apache.flink.ml.pipeline.{FitOperation, PredictOperation, Predictor}
-
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Random
-
-import breeze.linalg.{DenseVector => BreezeDenseVector, Vector => BreezeVector}
-
-/** Implements a soft-margin SVM using the communication-efficient distributed dual coordinate
- * ascent algorithm (CoCoA) with hinge-loss function.
- *
- * It can be used for binary classification problems, with the labels set as +1.0 to indicate a
- * positive example and -1.0 to indicate a negative example.
- *
- * The algorithm solves the following minimization problem:
- *
- * `min_{w in bbb"R"^d} lambda/2 ||w||^2 + 1/n sum_(i=1)^n l_{i}(w^Tx_i)`
- *
- * with `w` being the weight vector, `lambda` being the regularization constant,
- * `x_{i} in bbb"R"^d` being the data points and `l_{i}` being the convex loss functions, which
- * can also depend on the labels `y_{i} in bbb"R"`.
- * In the current implementation the regularizer is the 2-norm and the loss functions are the
- * hinge-loss functions:
- *
- * `l_{i} = max(0, 1 - y_{i} * w^Tx_i`
- *
- * With these choices, the problem definition is equivalent to a SVM with soft-margin.
- * Thus, the algorithm allows us to train a SVM with soft-margin.
- *
- * The minimization problem is solved by applying stochastic dual coordinate ascent (SDCA).
- * In order to make the algorithm efficient in a distributed setting, the CoCoA algorithm
- * calculates several iterations of SDCA locally on a data block before merging the local
- * updates into a valid global state.
- * This state is redistributed to the different data partitions where the next round of local
- * SDCA iterations is then executed.
- * The number of outer iterations and local SDCA iterations control the overall network costs,
- * because there is only network communication required for each outer iteration.
- * The local SDCA iterations are embarrassingly parallel once the individual data partitions have
- * been distributed across the cluster.
- *
- * Further details of the algorithm can be found [[http://arxiv.org/abs/1409.1458 here]].
- *
- * @example
- * {{{
- * val trainingDS: DataSet[LabeledVector] = env.readLibSVM(pathToTrainingFile)
- *
- * val svm = SVM()
- * .setBlocks(10)
- *
- * svm.fit(trainingDS)
- *
- * val testingDS: DataSet[Vector] = env.readLibSVM(pathToTestingFile)
- * .map(lv => lv.vector)
- *
- * val predictionDS: DataSet[(Vector, Double)] = svm.predict(testingDS)
- * }}}
- *
- * =Parameters=
- *
- * - [[org.apache.flink.ml.classification.SVM.Blocks]]:
- * Sets the number of blocks into which the input data will be split. On each block the local
- * stochastic dual coordinate ascent method is executed. This number should be set at least to
- * the degree of parallelism. If no value is specified, then the parallelism of the input
- * [[DataSet]] is used as the number of blocks. (Default value: '''None''')
- *
- * - [[org.apache.flink.ml.classification.SVM.Iterations]]:
- * Defines the maximum number of iterations of the outer loop method. In other words, it defines
- * how often the SDCA method is applied to the blocked data. After each iteration, the locally
- * computed weight vector updates have to be reduced to update the global weight vector value.
- * The new weight vector is broadcast to all SDCA tasks at the beginning of each iteration.
- * (Default value: '''10''')
- *
- * - [[org.apache.flink.ml.classification.SVM.LocalIterations]]:
- * Defines the maximum number of SDCA iterations. In other words, it defines how many data points
- * are drawn from each local data block to calculate the stochastic dual coordinate ascent.
- * (Default value: '''10''')
- *
- * - [[org.apache.flink.ml.classification.SVM.Regularization]]:
- * Defines the regularization constant of the SVM algorithm. The higher the value, the smaller
- * will the 2-norm of the weight vector be. In case of a SVM with hinge loss this means that the
- * SVM margin will be wider even though it might contain some false classifications.
- * (Default value: '''1.0''')
- *
- * - [[org.apache.flink.ml.classification.SVM.Stepsize]]:
- * Defines the initial step size for the updates of the weight vector. The larger the step size
- * is, the larger will be the contribution of the weight vector updates to the next weight vector
- * value. The effective scaling of the updates is `stepsize/blocks`. This value has to be tuned
- * in case that the algorithm becomes instable. (Default value: '''1.0''')
- *
- * - [[org.apache.flink.ml.classification.SVM.Seed]]:
- * Defines the seed to initialize the random number generator. The seed directly controls which
- * data points are chosen for the SDCA method. (Default value: '''Random value''')
- *
- * - [[org.apache.flink.ml.classification.SVM.ThresholdValue]]:
- * Defines the limiting value for the decision function above which examples are labeled as
- * positive (+1.0). Examples with a decision function value below this value are classified as
- * negative(-1.0). In order to get the raw decision function values you need to indicate it by
- * using the [[org.apache.flink.ml.classification.SVM.OutputDecisionFunction]].
- * (Default value: '''0.0''')
- *
- * - [[org.apache.flink.ml.classification.SVM.OutputDecisionFunction]]:
- * Determines whether the predict and evaluate functions of the SVM should return the distance
- * to the separating hyperplane, or binary class labels. Setting this to true will return the raw
- * distance to the hyperplane for each example. Setting it to false will return the binary
- * class label (+1.0, -1.0) (Default value: '''false''')
- */
-class SVM extends Predictor[SVM] {
-
- import SVM._
-
- /** Stores the learned weight vector after the fit operation */
- var weightsOption: Option[DataSet[DenseVector]] = None
-
- /** Sets the number of data blocks/partitions
- *
- * @param blocks the number of blocks into which the input data will be split.
- * @return itself
- */
- def setBlocks(blocks: Int): SVM = {
- parameters.add(Blocks, blocks)
- this
- }
-
- /** Sets the number of outer iterations
- *
- * @param iterations the maximum number of iterations of the outer loop method
- * @return itself
- */
- def setIterations(iterations: Int): SVM = {
- parameters.add(Iterations, iterations)
- this
- }
-
- /** Sets the number of local SDCA iterations
- *
- * @param localIterations the maximum number of SDCA iterations
- * @return itself
- */
- def setLocalIterations(localIterations: Int): SVM = {
- parameters.add(LocalIterations, localIterations)
- this
- }
-
- /** Sets the regularization constant
- *
- * @param regularization the regularization constant of the SVM algorithm
- * @return itself
- */
- def setRegularization(regularization: Double): SVM = {
- parameters.add(Regularization, regularization)
- this
- }
-
- /** Sets the stepsize for the weight vector updates
- *
- * @param stepsize the initial step size for the updates of the weight vector
- * @return itself
- */
- def setStepsize(stepsize: Double): SVM = {
- parameters.add(Stepsize, stepsize)
- this
- }
-
- /** Sets the seed value for the random number generator
- *
- * @param seed the seed to initialize the random number generator
- * @return itself
- */
- def setSeed(seed: Long): SVM = {
- parameters.add(Seed, seed)
- this
- }
-
- /** Sets the threshold above which elements are classified as positive.
- *
- * The [[predict ]] and [[evaluate]] functions will return +1.0 for items with a decision
- * function value above this threshold, and -1.0 for items below it.
- * @param threshold the limiting value for the decision function above which examples are
- * labeled as positive
- * @return itself
- */
- def setThreshold(threshold: Double): SVM = {
- parameters.add(ThresholdValue, threshold)
- this
- }
-
- /** Sets whether the predictions should return the raw decision function value or the
- * thresholded binary value.
- *
- * When setting this to true, predict and evaluate return the raw decision value, which is
- * the distance from the separating hyperplane.
- * When setting this to false, they return thresholded (+1.0, -1.0) values.
- *
- * @param outputDecisionFunction When set to true, [[predict ]] and [[evaluate]] return the raw
- * decision function values. When set to false, they return the
- * thresholded binary values (+1.0, -1.0).
- * @return itself
- */
- def setOutputDecisionFunction(outputDecisionFunction: Boolean): SVM = {
- parameters.add(OutputDecisionFunction, outputDecisionFunction)
- this
- }
-}
-
-/** Companion object of SVM. Contains convenience functions and the parameter type definitions
- * of the algorithm.
- */
-object SVM{
-
- val WEIGHT_VECTOR_BROADCAST_NAME = "weightVector"
-
- // ========================================== Parameters =========================================
-
- case object Blocks extends Parameter[Int] {
- val defaultValue: Option[Int] = None
- }
-
- case object Iterations extends Parameter[Int] {
- val defaultValue = Some(10)
- }
-
- case object LocalIterations extends Parameter[Int] {
- val defaultValue = Some(10)
- }
-
- case object Regularization extends Parameter[Double] {
- val defaultValue = Some(1.0)
- }
-
- case object Stepsize extends Parameter[Double] {
- val defaultValue = Some(1.0)
- }
-
- case object Seed extends Parameter[Long] {
- val defaultValue = Some(Random.nextLong())
- }
-
- case object ThresholdValue extends Parameter[Double] {
- val defaultValue = Some(0.0)
- }
-
- case object OutputDecisionFunction extends Parameter[Boolean] {
- val defaultValue = Some(false)
- }
-
- // ========================================== Factory methods ====================================
-
- def apply(): SVM = {
- new SVM()
- }
-
- // ========================================== Operations =========================================
-
- /** Provides the operation that makes the predictions for individual examples.
- *
- * @tparam T Input data type which is a subtype of [[Vector]]
- * @return A PredictOperation, through which it is possible to predict a value, given a
- * feature vector
- */
- implicit def predictVectors[T <: Vector] = {
- new PredictOperation[SVM, DenseVector, T, Double](){
-
- var thresholdValue: Double = _
- var outputDecisionFunction: Boolean = _
-
- override def getModel(self: SVM, predictParameters: ParameterMap): DataSet[DenseVector] = {
- thresholdValue = predictParameters(ThresholdValue)
- outputDecisionFunction = predictParameters(OutputDecisionFunction)
- self.weightsOption match {
- case Some(model) => model
- case None => {
- throw new RuntimeException("The SVM model has not been trained. Call first fit" +
- "before calling the predict operation.")
- }
- }
- }
-
- override def predict(value: T, model: DenseVector): Double = {
- val rawValue = value.asBreeze dot model.asBreeze
-
- if (outputDecisionFunction) {
- rawValue
- } else {
- if (rawValue > thresholdValue) 1.0 else -1.0
- }
- }
- }
- }
-
- /** [[FitOperation]] which trains a SVM with soft-margin based on the given training data set.
- *
- */
- implicit val fitSVM = {
- new FitOperation[SVM, LabeledVector] {
- override def fit(
- instance: SVM,
- fitParameters: ParameterMap,
- input: DataSet[LabeledVector])
- : Unit = {
- val resultingParameters = instance.parameters ++ fitParameters
-
- // Check if the number of blocks/partitions has been specified
- val blocks = resultingParameters.get(Blocks) match {
- case Some(value) => value
- case None => input.getParallelism
- }
-
- val scaling = resultingParameters(Stepsize)/blocks
- val iterations = resultingParameters(Iterations)
- val localIterations = resultingParameters(LocalIterations)
- val regularization = resultingParameters(Regularization)
- val seed = resultingParameters(Seed)
-
- // Obtain DataSet with the dimension of the data points
- val dimension = input.map{_.vector.size}.reduce{
- (a, b) => {
- require(a == b, "Dimensions of feature vectors have to be equal.")
- a
- }
- }
-
- val initialWeights = createInitialWeights(dimension)
-
- // Count the number of vectors, but keep the value in a DataSet to broadcast it later
- // TODO: Once efficient count and intermediate result partitions are implemented, use count
- val numberVectors = input map { x => 1 } reduce { _ + _ }
-
- // Group the input data into blocks in round robin fashion
- val blockedInputNumberElements = FlinkMLTools.block(
- input,
- blocks,
- Some(ModuloKeyPartitioner)).
- cross(numberVectors).
- map { x => x }
-
- val resultingWeights = initialWeights.iterate(iterations) {
- weights => {
- // compute the local SDCA to obtain the weight vector updates
- val deltaWs = localDualMethod(
- weights,
- blockedInputNumberElements,
- localIterations,
- regularization,
- scaling,
- seed
- )
-
- // scale the weight vectors
- val weightedDeltaWs = deltaWs map {
- deltaW => {
- deltaW :*= scaling
- }
- }
-
- // calculate the new weight vector by adding the weight vector updates to the weight
- // vector value
- weights.union(weightedDeltaWs).reduce { _ + _ }
- }
- }
-
- // Store the learned weight vector in hte given instance
- instance.weightsOption = Some(resultingWeights.map(_.fromBreeze[DenseVector]))
- }
- }
- }
-
- /** Creates a zero vector of length dimension
- *
- * @param dimension [[DataSet]] containing the dimension of the initial weight vector
- * @return Zero vector of length dimension
- */
- private def createInitialWeights(dimension: DataSet[Int]): DataSet[BreezeDenseVector[Double]] = {
- dimension.map {
- d => BreezeDenseVector.zeros[Double](d)
- }
- }
-
- /** Computes the local SDCA on the individual data blocks/partitions
- *
- * @param w Current weight vector
- * @param blockedInputNumberElements Blocked/Partitioned input data
- * @param localIterations Number of local SDCA iterations
- * @param regularization Regularization constant
- * @param scaling Scaling value for new weight vector updates
- * @param seed Random number generator seed
- * @return [[DataSet]] of weight vector updates. The weight vector updates are double arrays
- */
- private def localDualMethod(
- w: DataSet[BreezeDenseVector[Double]],
- blockedInputNumberElements: DataSet[(Block[LabeledVector], Int)],
- localIterations: Int,
- regularization: Double,
- scaling: Double,
- seed: Long)
- : DataSet[BreezeDenseVector[Double]] = {
- /*
- Rich mapper calculating for each data block the local SDCA. We use a RichMapFunction here,
- because we broadcast the current value of the weight vector to all mappers.
- */
- val localSDCA = new RichMapFunction[(Block[LabeledVector], Int), BreezeDenseVector[Double]] {
- var originalW: BreezeDenseVector[Double] = _
- // we keep the alphas across the outer loop iterations
- val alphasArray = ArrayBuffer[BreezeDenseVector[Double]]()
- // there might be several data blocks in one Flink partition, therefore store mapping
- val idMapping = scala.collection.mutable.HashMap[Int, Int]()
- var counter = 0
-
- var r: Random = _
-
- override def open(parameters: Configuration): Unit = {
- originalW = getRuntimeContext.getBroadcastVariable(WEIGHT_VECTOR_BROADCAST_NAME).get(0)
-
- if(r == null){
- r = new Random(seed ^ getRuntimeContext.getIndexOfThisSubtask)
- }
- }
-
- override def map(blockNumberElements: (Block[LabeledVector], Int))
- : BreezeDenseVector[Double] = {
- val (block, numberElements) = blockNumberElements
-
- // check if we already processed a data block with the corresponding block index
- val localIndex = idMapping.get(block.index) match {
- case Some(idx) => idx
- case None =>
- idMapping += (block.index -> counter)
- counter += 1
-
- alphasArray += BreezeDenseVector.zeros[Double](block.values.length)
-
- counter - 1
- }
-
- // create temporary alpha array for the local SDCA iterations
- val tempAlphas = alphasArray(localIndex).copy
-
- val numLocalDatapoints = tempAlphas.length
- val deltaAlphas = BreezeDenseVector.zeros[Double](numLocalDatapoints)
-
- val w = originalW.copy
-
- val deltaW = BreezeDenseVector.zeros[Double](originalW.length)
-
- for(i <- 1 to localIterations) {
- // pick random data point for SDCA
- val idx = r.nextInt(numLocalDatapoints)
-
- val LabeledVector(label, vector) = block.values(idx)
- val alpha = tempAlphas(idx)
-
- // maximize the dual problem and retrieve alpha and weight vector updates
- val (deltaAlpha, deltaWUpdate) = maximize(
- vector.asBreeze,
- label,
- regularization,
- alpha,
- w,
- numberElements)
-
- // update alpha values
- tempAlphas(idx) += deltaAlpha
- deltaAlphas(idx) += deltaAlpha
-
- // deltaWUpdate is already scaled with 1/lambda/n
- w += deltaWUpdate
- deltaW += deltaWUpdate
- }
-
- // update local alpha values
- alphasArray(localIndex) += deltaAlphas * scaling
-
- deltaW
- }
- }
-
- blockedInputNumberElements.map(localSDCA).withBroadcastSet(w, WEIGHT_VECTOR_BROADCAST_NAME)
- }
-
- /** Maximizes the dual problem using hinge loss functions. It returns the alpha and weight
- * vector updates.
- *
- * @param x Selected data point
- * @param y Label of selected data point
- * @param regularization Regularization constant
- * @param alpha Alpha value of selected data point
- * @param w Current weight vector value
- * @param numberElements Number of elements in the training data set
- * @return Alpha and weight vector updates
- */
- private def maximize(
- x: BreezeVector[Double],
- y: Double, regularization: Double,
- alpha: Double,
- w: BreezeVector[Double],
- numberElements: Int)
- : (Double, BreezeVector[Double]) = {
- // compute hinge loss gradient
- val dotProduct = x dot w
- val grad = (y * dotProduct - 1.0) * (regularization * numberElements)
-
- // compute projected gradient
- var proj_grad = if(alpha <= 0.0){
- scala.math.min(grad, 0)
- } else if(alpha >= 1.0) {
- scala.math.max(grad, 0)
- } else {
- grad
- }
-
- if(scala.math.abs(grad) != 0.0){
- val qii = x dot x
- val newAlpha = if(qii != 0.0){
- scala.math.min(scala.math.max(alpha - (grad / qii), 0.0), 1.0)
- } else {
- 1.0
- }
-
- val deltaW = x * y * (newAlpha - alpha) / (regularization * numberElements)
-
- (newAlpha - alpha, deltaW)
- } else {
- (0.0 , BreezeVector.zeros(w.length))
- }
- }
-
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/Block.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/Block.scala
deleted file mode 100644
index 9c55534..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/Block.scala
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.common
-
-/** Base class for blocks of elements.
- *
- * TODO: Replace Vector type by Array type once Flink supports generic arrays
- *
- * @param index
- * @param values
- * @tparam T
- */
-case class Block[T](index: Int, values: Vector[T])
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/FlinkMLTools.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/FlinkMLTools.scala
deleted file mode 100644
index 4bb21d5..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/FlinkMLTools.scala
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.common
-
-import org.apache.flink.api.common.functions.Partitioner
-import org.apache.flink.api.common.io.FileOutputFormat.OutputDirectoryMode
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.io.{TypeSerializerInputFormat, TypeSerializerOutputFormat}
-import org.apache.flink.api.scala._
-import org.apache.flink.core.fs.FileSystem.WriteMode
-import org.apache.flink.core.fs.Path
-
-import scala.reflect.ClassTag
-
-/** FlinkMLTools contains a set of convenience functions for Flink's machine learning library:
- *
- * - persist:
- * Takes up to 5 [[DataSet]]s and file paths. Each [[DataSet]] is written to the specified
- * path and subsequently re-read from disk. This method can be used to effectively split the
- * execution graph at the given [[DataSet]]. Writing it to disk triggers its materialization
- * and specifying it as a source will prevent the re-execution of it.
- *
- * - block:
- * Takes a DataSet of elements T and groups them in n blocks.
- *
- */
-object FlinkMLTools {
- val EXECUTION_ENVIRONMENT_NAME = "FlinkMLTools persist"
-
- /** Registers the different FlinkML related types for Kryo serialization
- *
- * @param env The Flink execution environment where the types need to be registered
- */
- def registerFlinkMLTypes(env: ExecutionEnvironment): Unit = {
-
- // Vector types
- env.registerType(classOf[org.apache.flink.ml.math.DenseVector])
- env.registerType(classOf[org.apache.flink.ml.math.SparseVector])
-
- // Matrix types
- env.registerType(classOf[org.apache.flink.ml.math.DenseMatrix])
- env.registerType(classOf[org.apache.flink.ml.math.SparseMatrix])
-
- // Breeze Vector types
- env.registerType(classOf[breeze.linalg.DenseVector[_]])
- env.registerType(classOf[breeze.linalg.SparseVector[_]])
-
- // Breeze specialized types
- env.registerType(breeze.linalg.DenseVector.zeros[Double](0).getClass)
- env.registerType(breeze.linalg.SparseVector.zeros[Double](0).getClass)
-
- // Breeze Matrix types
- env.registerType(classOf[breeze.linalg.DenseMatrix[Double]])
- env.registerType(classOf[breeze.linalg.CSCMatrix[Double]])
-
- // Breeze specialized types
- env.registerType(breeze.linalg.DenseMatrix.zeros[Double](0, 0).getClass)
- env.registerType(breeze.linalg.CSCMatrix.zeros[Double](0, 0).getClass)
- }
-
- /** Writes a [[DataSet]] to the specified path and returns it as a DataSource for subsequent
- * operations.
- *
- * @param dataset [[DataSet]] to write to disk
- * @param path File path to write dataset to
- * @tparam T Type of the [[DataSet]] elements
- * @return [[DataSet]] reading the just written file
- */
- def persist[T: ClassTag: TypeInformation](dataset: DataSet[T], path: String): DataSet[T] = {
- val env = dataset.getExecutionEnvironment
- val outputFormat = new TypeSerializerOutputFormat[T]
-
- val filePath = new Path(path)
-
- outputFormat.setOutputFilePath(filePath)
- outputFormat.setWriteMode(WriteMode.OVERWRITE)
-
- dataset.output(outputFormat)
- env.execute(EXECUTION_ENVIRONMENT_NAME)
-
- val inputFormat = new TypeSerializerInputFormat[T](dataset.getType)
- inputFormat.setFilePath(filePath)
-
- env.createInput(inputFormat)
- }
-
- /** Writes multiple [[DataSet]]s to the specified paths and returns them as DataSources for
- * subsequent operations.
- *
- * @param ds1 First [[DataSet]] to write to disk
- * @param ds2 Second [[DataSet]] to write to disk
- * @param path1 Path for ds1
- * @param path2 Path for ds2
- * @tparam A Type of the first [[DataSet]]'s elements
- * @tparam B Type of the second [[DataSet]]'s elements
- * @return Tuple of [[DataSet]]s reading the just written files
- */
- def persist[A: ClassTag: TypeInformation ,B: ClassTag: TypeInformation](ds1: DataSet[A], ds2:
- DataSet[B], path1: String, path2: String): (DataSet[A], DataSet[B]) = {
- val env = ds1.getExecutionEnvironment
-
- val f1 = new Path(path1)
-
- val of1 = new TypeSerializerOutputFormat[A]
- of1.setOutputFilePath(f1)
- of1.setWriteMode(WriteMode.OVERWRITE)
-
- ds1.output(of1)
-
- val f2 = new Path(path2)
-
- val of2 = new TypeSerializerOutputFormat[B]
- of2.setOutputFilePath(f2)
- of2.setWriteMode(WriteMode.OVERWRITE)
-
- ds2.output(of2)
-
- env.execute(EXECUTION_ENVIRONMENT_NAME)
-
- val if1 = new TypeSerializerInputFormat[A](ds1.getType)
- if1.setFilePath(f1)
-
- val if2 = new TypeSerializerInputFormat[B](ds2.getType)
- if2.setFilePath(f2)
-
- (env.createInput(if1), env.createInput(if2))
- }
-
- /** Writes multiple [[DataSet]]s to the specified paths and returns them as DataSources for
- * subsequent operations.
- *
- * @param ds1 First [[DataSet]] to write to disk
- * @param ds2 Second [[DataSet]] to write to disk
- * @param ds3 Third [[DataSet]] to write to disk
- * @param path1 Path for ds1
- * @param path2 Path for ds2
- * @param path3 Path for ds3
- * @tparam A Type of first [[DataSet]]'s elements
- * @tparam B Type of second [[DataSet]]'s elements
- * @tparam C Type of third [[DataSet]]'s elements
- * @return Tuple of [[DataSet]]s reading the just written files
- */
- def persist[A: ClassTag: TypeInformation ,B: ClassTag: TypeInformation,
- C: ClassTag: TypeInformation](ds1: DataSet[A], ds2: DataSet[B], ds3: DataSet[C], path1:
- String, path2: String, path3: String): (DataSet[A], DataSet[B], DataSet[C]) = {
- val env = ds1.getExecutionEnvironment
-
- val f1 = new Path(path1)
-
- val of1 = new TypeSerializerOutputFormat[A]
- of1.setOutputFilePath(f1)
- of1.setWriteMode(WriteMode.OVERWRITE)
-
- ds1.output(of1)
-
- val f2 = new Path(path2)
-
- val of2 = new TypeSerializerOutputFormat[B]
- of2.setOutputFilePath(f2)
- of2.setWriteMode(WriteMode.OVERWRITE)
-
- ds2.output(of2)
-
- val f3 = new Path(path3)
-
- val of3 = new TypeSerializerOutputFormat[C]
- of3.setOutputFilePath(f3)
- of3.setWriteMode(WriteMode.OVERWRITE)
-
- ds3.output(of3)
-
- env.execute(EXECUTION_ENVIRONMENT_NAME)
-
- val if1 = new TypeSerializerInputFormat[A](ds1.getType)
- if1.setFilePath(f1)
-
- val if2 = new TypeSerializerInputFormat[B](ds2.getType)
- if2.setFilePath(f2)
-
- val if3 = new TypeSerializerInputFormat[C](ds3.getType)
- if3.setFilePath(f3)
-
- (env.createInput(if1), env.createInput(if2), env.createInput(if3))
- }
-
- /** Writes multiple [[DataSet]]s to the specified paths and returns them as DataSources for
- * subsequent operations.
- *
- * @param ds1 First [[DataSet]] to write to disk
- * @param ds2 Second [[DataSet]] to write to disk
- * @param ds3 Third [[DataSet]] to write to disk
- * @param ds4 Fourth [[DataSet]] to write to disk
- * @param path1 Path for ds1
- * @param path2 Path for ds2
- * @param path3 Path for ds3
- * @param path4 Path for ds4
- * @tparam A Type of first [[DataSet]]'s elements
- * @tparam B Type of second [[DataSet]]'s elements
- * @tparam C Type of third [[DataSet]]'s elements
- * @tparam D Type of fourth [[DataSet]]'s elements
- * @return Tuple of [[DataSet]]s reading the just written files
- */
- def persist[A: ClassTag: TypeInformation ,B: ClassTag: TypeInformation,
- C: ClassTag: TypeInformation, D: ClassTag: TypeInformation](ds1: DataSet[A], ds2: DataSet[B],
- ds3: DataSet[C], ds4: DataSet[D],
- path1: String, path2: String, path3:
- String, path4: String):
- (DataSet[A], DataSet[B], DataSet[C], DataSet[D]) = {
- val env = ds1.getExecutionEnvironment
-
- val f1 = new Path(path1)
-
- val of1 = new TypeSerializerOutputFormat[A]
- of1.setOutputFilePath(f1)
- of1.setWriteMode(WriteMode.OVERWRITE)
-
- ds1.output(of1)
-
- val f2 = new Path(path2)
-
- val of2 = new TypeSerializerOutputFormat[B]
- of2.setOutputFilePath(f2)
- of2.setWriteMode(WriteMode.OVERWRITE)
-
- ds2.output(of2)
-
- val f3 = new Path(path3)
-
- val of3 = new TypeSerializerOutputFormat[C]
- of3.setOutputFilePath(f3)
- of3.setWriteMode(WriteMode.OVERWRITE)
-
- ds3.output(of3)
-
- val f4 = new Path(path4)
-
- val of4 = new TypeSerializerOutputFormat[D]
- of4.setOutputFilePath(f4)
- of4.setWriteMode(WriteMode.OVERWRITE)
-
- ds4.output(of4)
-
- env.execute(EXECUTION_ENVIRONMENT_NAME)
-
- val if1 = new TypeSerializerInputFormat[A](ds1.getType)
- if1.setFilePath(f1)
-
- val if2 = new TypeSerializerInputFormat[B](ds2.getType)
- if2.setFilePath(f2)
-
- val if3 = new TypeSerializerInputFormat[C](ds3.getType)
- if3.setFilePath(f3)
-
- val if4 = new TypeSerializerInputFormat[D](ds4.getType)
- if4.setFilePath(f4)
-
- (env.createInput(if1), env.createInput(if2), env.createInput(if3), env.createInput(if4))
- }
-
- /** Writes multiple [[DataSet]]s to the specified paths and returns them as DataSources for
- * subsequent operations.
- *
- * @param ds1 First [[DataSet]] to write to disk
- * @param ds2 Second [[DataSet]] to write to disk
- * @param ds3 Third [[DataSet]] to write to disk
- * @param ds4 Fourth [[DataSet]] to write to disk
- * @param ds5 Fifth [[DataSet]] to write to disk
- * @param path1 Path for ds1
- * @param path2 Path for ds2
- * @param path3 Path for ds3
- * @param path4 Path for ds4
- * @param path5 Path for ds5
- * @tparam A Type of first [[DataSet]]'s elements
- * @tparam B Type of second [[DataSet]]'s elements
- * @tparam C Type of third [[DataSet]]'s elements
- * @tparam D Type of fourth [[DataSet]]'s elements
- * @tparam E Type of fifth [[DataSet]]'s elements
- * @return Tuple of [[DataSet]]s reading the just written files
- */
- def persist[A: ClassTag: TypeInformation ,B: ClassTag: TypeInformation,
- C: ClassTag: TypeInformation, D: ClassTag: TypeInformation, E: ClassTag: TypeInformation]
- (ds1: DataSet[A], ds2: DataSet[B], ds3: DataSet[C], ds4: DataSet[D], ds5: DataSet[E], path1:
- String, path2: String, path3: String, path4: String, path5: String): (DataSet[A], DataSet[B],
- DataSet[C], DataSet[D], DataSet[E]) = {
- val env = ds1.getExecutionEnvironment
-
- val f1 = new Path(path1)
-
- val of1 = new TypeSerializerOutputFormat[A]
- of1.setOutputFilePath(f1)
- of1.setWriteMode(WriteMode.OVERWRITE)
-
- ds1.output(of1)
-
- val f2 = new Path(path2)
-
- val of2 = new TypeSerializerOutputFormat[B]
- of2.setOutputFilePath(f2)
- of2.setOutputDirectoryMode(OutputDirectoryMode.ALWAYS)
- of2.setWriteMode(WriteMode.OVERWRITE)
-
- ds2.output(of2)
-
- val f3 = new Path(path3)
-
- val of3 = new TypeSerializerOutputFormat[C]
- of3.setOutputFilePath(f3)
- of3.setWriteMode(WriteMode.OVERWRITE)
-
- ds3.output(of3)
-
- val f4 = new Path(path4)
-
- val of4 = new TypeSerializerOutputFormat[D]
- of4.setOutputFilePath(f4)
- of4.setWriteMode(WriteMode.OVERWRITE)
-
- ds4.output(of4)
-
- val f5 = new Path(path5)
-
- val of5 = new TypeSerializerOutputFormat[E]
- of5.setOutputFilePath(f5)
- of5.setWriteMode(WriteMode.OVERWRITE)
-
- ds5.output(of5)
-
- env.execute(EXECUTION_ENVIRONMENT_NAME)
-
- val if1 = new TypeSerializerInputFormat[A](ds1.getType)
- if1.setFilePath(f1)
-
- val if2 = new TypeSerializerInputFormat[B](ds2.getType)
- if2.setFilePath(f2)
-
- val if3 = new TypeSerializerInputFormat[C](ds3.getType)
- if3.setFilePath(f3)
-
- val if4 = new TypeSerializerInputFormat[D](ds4.getType)
- if4.setFilePath(f4)
-
- val if5 = new TypeSerializerInputFormat[E](ds5.getType)
- if5.setFilePath(f5)
-
- (env.createInput(if1), env.createInput(if2), env.createInput(if3), env.createInput(if4), env
- .createInput(if5))
- }
-
- /** Groups the DataSet input into numBlocks blocks.
- *
- * @param input the input dataset
- * @param numBlocks Number of Blocks
- * @param partitionerOption Optional partitioner to control the partitioning
- * @tparam T Type of the [[DataSet]]'s elements
- * @return The different datasets grouped into blocks
- */
- def block[T: TypeInformation: ClassTag](
- input: DataSet[T],
- numBlocks: Int,
- partitionerOption: Option[Partitioner[Int]] = None)
- : DataSet[Block[T]] = {
- val blockIDInput = input map {
- element: T =>
- val blockID = element.hashCode() % numBlocks
-
- val blockIDResult = if(blockID < 0){
- blockID + numBlocks
- } else {
- blockID
- }
-
- (blockIDResult, element)
- }
-
- val preGroupBlockIDInput = partitionerOption match {
- case Some(partitioner) =>
- blockIDInput partitionCustom(partitioner, 0)
-
- case None => blockIDInput
- }
-
- preGroupBlockIDInput.groupBy(0).reduceGroup {
- iterator => {
- val array = iterator.toVector
-
- val blockID = array.head._1
- val elements = array.map(_._2)
-
- Block[T](blockID, elements)
- }
- }.withForwardedFields("0 -> index")
- }
-
- /** Distributes the elements by taking the modulo of their keys and assigning it to this channel
- *
- */
- object ModuloKeyPartitioner extends Partitioner[Int] {
- override def partition(key: Int, numPartitions: Int): Int = {
- val result = key % numPartitions
-
- if(result < 0) {
- result + numPartitions
- } else {
- result
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/LabeledVector.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/LabeledVector.scala
deleted file mode 100644
index 3b948c0..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/LabeledVector.scala
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.common
-
-import org.apache.flink.ml.math.Vector
-
-/** This class represents a vector with an associated label as it is required for many supervised
- * learning tasks.
- *
- * @param label Label of the data point
- * @param vector Data point
- */
-case class LabeledVector(label: Double, vector: Vector) extends Serializable {
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case labeledVector: LabeledVector =>
- vector.equals(labeledVector.vector) && label.equals(labeledVector.label)
- case _ => false
- }
- }
-
- override def toString: String = {
- s"LabeledVector($label, $vector)"
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/ParameterMap.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/ParameterMap.scala
deleted file mode 100644
index 77d2d46..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/ParameterMap.scala
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.common
-
-import scala.collection.mutable
-
-/**
- * Map used to store configuration parameters for algorithms. The parameter
- * values are stored in a [[Map]] being identified by a [[Parameter]] object. ParameterMaps can
- * be fused. This operation is left associative, meaning that latter ParameterMaps can override
- * parameter values defined in a preceding ParameterMap.
- *
- * @param map Map containing parameter settings
- */
-class ParameterMap(val map: mutable.Map[Parameter[_], Any]) extends Serializable {
-
- def this() = {
- this(new mutable.HashMap[Parameter[_], Any]())
- }
-
- /**
- * Adds a new parameter value to the ParameterMap.
- *
- * @param parameter Key
- * @param value Value associated with the given key
- * @tparam T Type of value
- */
- def add[T](parameter: Parameter[T], value: T): ParameterMap = {
- map += (parameter -> value)
- this
- }
-
- /**
- * Retrieves a parameter value associated to a given key. The value is returned as an Option.
- * If there is no value associated to the given key, then the default value of the [[Parameter]]
- * is returned.
- *
- * @param parameter Key
- * @tparam T Type of the value to retrieve
- * @return Some(value) if an value is associated to the given key, otherwise the default value
- * defined by parameter
- */
- def get[T](parameter: Parameter[T]): Option[T] = {
- if(map.isDefinedAt(parameter)) {
- map.get(parameter).asInstanceOf[Option[T]]
- } else {
- parameter.defaultValue
- }
- }
-
- /**
- * Retrieves a parameter value associated to a given key. If there is no value contained in the
- * map, then the default value of the [[Parameter]] is checked. If the default value is defined,
- * then it is returned. If the default is undefined, then a [[NoSuchElementException]] is thrown.
- *
- * @param parameter Key
- * @tparam T Type of value
- * @return Value associated with the given key or its default value
- */
- def apply[T](parameter: Parameter[T]): T = {
- if(map.isDefinedAt(parameter)) {
- map(parameter).asInstanceOf[T]
- } else {
- parameter.defaultValue match {
- case Some(value) => value
- case None => throw new NoSuchElementException(s"Could not retrieve " +
- s"parameter value $parameter.")
- }
- }
- }
-
- /**
- * Adds the parameter values contained in parameters to itself.
- *
- * @param parameters [[ParameterMap]] containing the parameter values to be added
- * @return this after inserting the parameter values from parameters
- */
- def ++(parameters: ParameterMap): ParameterMap = {
- val result = new ParameterMap(map)
- result.map ++= parameters.map
-
- result
- }
-}
-
-object ParameterMap {
- val Empty = new ParameterMap
-
- def apply(): ParameterMap = {
- new ParameterMap
- }
-}
-
-/**
- * Base trait for parameter keys
- *
- * @tparam T Type of parameter value associated to this parameter key
- */
-trait Parameter[T] {
-
- /**
- * Default value of parameter. If no such value exists, then returns [[None]]
- */
- val defaultValue: Option[T]
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/WeightVector.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/WeightVector.scala
deleted file mode 100644
index 4628c71..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/WeightVector.scala
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.common
-
-import org.apache.flink.ml.math.Vector
-
-// TODO(tvas): This provides an abstraction for the weights
-// but at the same time it leads to the creation of many objects as we have to pack and unpack
-// the weights and the intercept often during SGD.
-
-/** This class represents a weight vector with an intercept, as it is required for many supervised
- * learning tasks
- * @param weights The vector of weights
- * @param intercept The intercept (bias) weight
- */
-case class WeightVector(weights: Vector, intercept: Double) extends Serializable {}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/WithParameters.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/WithParameters.scala
deleted file mode 100644
index 24ac9e3..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/common/WithParameters.scala
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.common
-
-/**
- * Adds a [[ParameterMap]] which can be used to store configuration values
- */
-trait WithParameters {
- val parameters = new ParameterMap
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/BLAS.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/BLAS.scala
deleted file mode 100644
index d134623..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/BLAS.scala
+++ /dev/null
@@ -1,291 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import com.github.fommil.netlib.{BLAS => NetlibBLAS, F2jBLAS}
-import com.github.fommil.netlib.BLAS.{getInstance => NativeBLAS}
-
-/**
- * BLAS routines for vectors and matrices.
- *
- * Original code from the Apache Spark project:
- * http://git.io/vfZUe
- */
-object BLAS extends Serializable {
-
- @transient private var _f2jBLAS: NetlibBLAS = _
- @transient private var _nativeBLAS: NetlibBLAS = _
-
- // For level-1 routines, we use Java implementation.
- private def f2jBLAS: NetlibBLAS = {
- if (_f2jBLAS == null) {
- _f2jBLAS = new F2jBLAS
- }
- _f2jBLAS
- }
-
- /**
- * y += a * x
- */
- def axpy(a: Double, x: Vector, y: Vector): Unit = {
- require(x.size == y.size)
- y match {
- case dy: DenseVector =>
- x match {
- case sx: SparseVector =>
- axpy(a, sx, dy)
- case dx: DenseVector =>
- axpy(a, dx, dy)
- case _ =>
- throw new UnsupportedOperationException(
- s"axpy doesn't support x type ${x.getClass}.")
- }
- case _ =>
- throw new IllegalArgumentException(
- s"axpy only supports adding to a dense vector but got type ${y.getClass}.")
- }
- }
-
- /**
- * y += a * x
- */
- private def axpy(a: Double, x: DenseVector, y: DenseVector): Unit = {
- val n = x.size
- f2jBLAS.daxpy(n, a, x.data, 1, y.data, 1)
- }
-
- /**
- * y += a * x
- */
- private def axpy(a: Double, x: SparseVector, y: DenseVector): Unit = {
- val xValues = x.data
- val xIndices = x.indices
- val yValues = y.data
- val nnz = xIndices.length
-
- if (a == 1.0) {
- var k = 0
- while (k < nnz) {
- yValues(xIndices(k)) += xValues(k)
- k += 1
- }
- } else {
- var k = 0
- while (k < nnz) {
- yValues(xIndices(k)) += a * xValues(k)
- k += 1
- }
- }
- }
-
- /**
- * dot(x, y)
- */
- def dot(x: Vector, y: Vector): Double = {
- require(x.size == y.size,
- "BLAS.dot(x: Vector, y:Vector) was given Vectors with non-matching sizes:" +
- " x.size = " + x.size + ", y.size = " + y.size)
- (x, y) match {
- case (dx: DenseVector, dy: DenseVector) =>
- dot(dx, dy)
- case (sx: SparseVector, dy: DenseVector) =>
- dot(sx, dy)
- case (dx: DenseVector, sy: SparseVector) =>
- dot(sy, dx)
- case (sx: SparseVector, sy: SparseVector) =>
- dot(sx, sy)
- case _ =>
- throw new IllegalArgumentException(s"dot doesn't support (${x.getClass}, ${y.getClass}).")
- }
- }
-
- /**
- * dot(x, y)
- */
- private def dot(x: DenseVector, y: DenseVector): Double = {
- val n = x.size
- f2jBLAS.ddot(n, x.data, 1, y.data, 1)
- }
-
- /**
- * dot(x, y)
- */
- private def dot(x: SparseVector, y: DenseVector): Double = {
- val xValues = x.data
- val xIndices = x.indices
- val yValues = y.data
- val nnz = xIndices.length
-
- var sum = 0.0
- var k = 0
- while (k < nnz) {
- sum += xValues(k) * yValues(xIndices(k))
- k += 1
- }
- sum
- }
-
- /**
- * dot(x, y)
- */
- private def dot(x: SparseVector, y: SparseVector): Double = {
- val xValues = x.data
- val xIndices = x.indices
- val yValues = y.data
- val yIndices = y.indices
- val nnzx = xIndices.length
- val nnzy = yIndices.length
-
- var kx = 0
- var ky = 0
- var sum = 0.0
- // y catching x
- while (kx < nnzx && ky < nnzy) {
- val ix = xIndices(kx)
- while (ky < nnzy && yIndices(ky) < ix) {
- ky += 1
- }
- if (ky < nnzy && yIndices(ky) == ix) {
- sum += xValues(kx) * yValues(ky)
- ky += 1
- }
- kx += 1
- }
- sum
- }
-
- /**
- * y = x
- */
- def copy(x: Vector, y: Vector): Unit = {
- val n = y.size
- require(x.size == n)
- y match {
- case dy: DenseVector =>
- x match {
- case sx: SparseVector =>
- val sxIndices = sx.indices
- val sxValues = sx.data
- val dyValues = dy.data
- val nnz = sxIndices.length
-
- var i = 0
- var k = 0
- while (k < nnz) {
- val j = sxIndices(k)
- while (i < j) {
- dyValues(i) = 0.0
- i += 1
- }
- dyValues(i) = sxValues(k)
- i += 1
- k += 1
- }
- while (i < n) {
- dyValues(i) = 0.0
- i += 1
- }
- case dx: DenseVector =>
- Array.copy(dx.data, 0, dy.data, 0, n)
- }
- case _ =>
- throw new IllegalArgumentException(s"y must be dense in copy but got ${y.getClass}")
- }
- }
-
- /**
- * x = a * x
- */
- def scal(a: Double, x: Vector): Unit = {
- x match {
- case sx: SparseVector =>
- f2jBLAS.dscal(sx.data.length, a, sx.data, 1)
- case dx: DenseVector =>
- f2jBLAS.dscal(dx.data.length, a, dx.data, 1)
- case _ =>
- throw new IllegalArgumentException(s"scal doesn't support vector type ${x.getClass}.")
- }
- }
-
- // For level-3 routines, we use the native BLAS.
- private def nativeBLAS: NetlibBLAS = {
- if (_nativeBLAS == null) {
- _nativeBLAS = NativeBLAS
- }
- _nativeBLAS
- }
-
- /**
- * A := alpha * x * x^T^ + A
- * @param alpha a real scalar that will be multiplied to x * x^T^.
- * @param x the vector x that contains the n elements.
- * @param A the symmetric matrix A. Size of n x n.
- */
- def syr(alpha: Double, x: Vector, A: DenseMatrix) {
- val mA = A.numRows
- val nA = A.numCols
- require(mA == nA, s"A is not a square matrix (and hence is not symmetric). A: $mA x $nA")
- require(mA == x.size, s"The size of x doesn't match the rank of A. A: $mA x $nA, x: ${x.size}")
-
- x match {
- case dv: DenseVector => syr(alpha, dv, A)
- case sv: SparseVector => syr(alpha, sv, A)
- case _ =>
- throw new IllegalArgumentException(s"syr doesn't support vector type ${x.getClass}.")
- }
- }
-
- private def syr(alpha: Double, x: DenseVector, A: DenseMatrix) {
- val nA = A.numRows
- val mA = A.numCols
-
- nativeBLAS.dsyr("U", x.size, alpha, x.data, 1, A.data, nA)
-
- // Fill lower triangular part of A
- var i = 0
- while (i < mA) {
- var j = i + 1
- while (j < nA) {
- A(j, i) = A(i, j)
- j += 1
- }
- i += 1
- }
- }
-
- private def syr(alpha: Double, x: SparseVector, A: DenseMatrix) {
- val mA = A.numCols
- val xIndices = x.indices
- val xValues = x.data
- val nnz = xValues.length
- val Avalues = A.data
-
- var i = 0
- while (i < nnz) {
- val multiplier = alpha * xValues(i)
- val offset = xIndices(i) * mA
- var j = 0
- while (j < nnz) {
- Avalues(xIndices(j) + offset) += multiplier * xValues(j)
- j += 1
- }
- i += 1
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Breeze.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Breeze.scala
deleted file mode 100644
index 5058ec3..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Breeze.scala
+++ /dev/null
@@ -1,88 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import breeze.linalg.{Matrix => BreezeMatrix, DenseMatrix => BreezeDenseMatrix,
-CSCMatrix => BreezeCSCMatrix, DenseVector => BreezeDenseVector, SparseVector => BreezeSparseVector,
-Vector => BreezeVector}
-
-/** This class contains convenience function to wrap a matrix/vector into a breeze matrix/vector
- * and to unwrap it again.
- *
- */
-object Breeze {
-
- implicit class Matrix2BreezeConverter(matrix: Matrix) {
- def asBreeze: BreezeMatrix[Double] = {
- matrix match {
- case dense: DenseMatrix =>
- new BreezeDenseMatrix[Double](
- dense.numRows,
- dense.numCols,
- dense.data)
-
- case sparse: SparseMatrix =>
- new BreezeCSCMatrix[Double](
- sparse.data,
- sparse.numRows,
- sparse.numCols,
- sparse.colPtrs,
- sparse.rowIndices
- )
- }
- }
- }
-
- implicit class Breeze2MatrixConverter(matrix: BreezeMatrix[Double]) {
- def fromBreeze: Matrix = {
- matrix match {
- case dense: BreezeDenseMatrix[Double] =>
- new DenseMatrix(dense.rows, dense.cols, dense.data)
-
- case sparse: BreezeCSCMatrix[Double] =>
- new SparseMatrix(sparse.rows, sparse.cols, sparse.rowIndices, sparse.colPtrs, sparse.data)
- }
- }
- }
-
- implicit class BreezeArrayConverter[T](array: Array[T]) {
- def asBreeze: BreezeDenseVector[T] = {
- new BreezeDenseVector[T](array)
- }
- }
-
- implicit class Breeze2VectorConverter(vector: BreezeVector[Double]) {
- def fromBreeze[T <: Vector: BreezeVectorConverter]: T = {
- val converter = implicitly[BreezeVectorConverter[T]]
- converter.convert(vector)
- }
- }
-
- implicit class Vector2BreezeConverter(vector: Vector) {
- def asBreeze: BreezeVector[Double] = {
- vector match {
- case dense: DenseVector =>
- new breeze.linalg.DenseVector(dense.data)
-
- case sparse: SparseVector =>
- new BreezeSparseVector(sparse.indices, sparse.data, sparse.size)
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/BreezeVectorConverter.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/BreezeVectorConverter.scala
deleted file mode 100644
index dac8fb2..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/BreezeVectorConverter.scala
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import breeze.linalg.{Vector => BreezeVector}
-
-/** Type class which allows the conversion from Breeze vectors to Flink vectors
- *
- * @tparam T Resulting type of the conversion, subtype of [[Vector]]
- */
-trait BreezeVectorConverter[T <: Vector] extends Serializable {
- /** Converts a Breeze vector into a Flink vector of type T
- *
- * @param vector Breeze vector
- * @return Flink vector of type T
- */
- def convert(vector: BreezeVector[Double]): T
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/DenseMatrix.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/DenseMatrix.scala
deleted file mode 100644
index 80d1dc3..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/DenseMatrix.scala
+++ /dev/null
@@ -1,191 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-/**
- * Dense matrix implementation of [[Matrix]]. Stores data in column major order in a continuous
- * double array.
- *
- * @param numRows Number of rows
- * @param numCols Number of columns
- * @param data Array of matrix elements in column major order
- */
-case class DenseMatrix(numRows: Int, numCols: Int, data: Array[Double])
- extends Matrix with Serializable {
-
- import DenseMatrix._
-
- require(numRows * numCols == data.length, s"The number of values ${data.length} does " +
- s"not correspond to its dimensions ($numRows, $numCols).")
-
- /**
- * Element wise access function
- *
- * @param row row index
- * @param col column index
- * @return matrix entry at (row, col)
- */
- override def apply(row: Int, col: Int): Double = {
- val index = locate(row, col)
-
- data(index)
- }
-
- override def toString: String = {
- val result = StringBuilder.newBuilder
- result.append(s"DenseMatrix($numRows, $numCols)\n")
-
- val columnsFieldWidths = for(row <- 0 until math.min(numRows, MAX_ROWS)) yield {
- var column = 0
- var maxFieldWidth = 0
-
- while(column * maxFieldWidth < LINE_WIDTH && column < numCols) {
- val fieldWidth = printEntry(row, column).length + 2
-
- if(fieldWidth > maxFieldWidth) {
- maxFieldWidth = fieldWidth
- }
-
- if(column * maxFieldWidth < LINE_WIDTH) {
- column += 1
- }
- }
-
- (column, maxFieldWidth)
- }
-
- val (columns, fieldWidths) = columnsFieldWidths.unzip
-
- val maxColumns = columns.min
- val fieldWidth = fieldWidths.max
-
- for(row <- 0 until math.min(numRows, MAX_ROWS)) {
- for(col <- 0 until maxColumns) {
- val str = printEntry(row, col)
-
- result.append(" " * (fieldWidth - str.length) + str)
- }
-
- if(maxColumns < numCols) {
- result.append("...")
- }
-
- result.append("\n")
- }
-
- if(numRows > MAX_ROWS) {
- result.append("...\n")
- }
-
- result.toString()
- }
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case dense: DenseMatrix =>
- numRows == dense.numRows && numCols == dense.numCols && data.sameElements(dense.data)
- case _ => false
- }
- }
-
- override def hashCode: Int = {
- val hashCodes = List(numRows.hashCode(), numCols.hashCode(), java.util.Arrays.hashCode(data))
-
- hashCodes.foldLeft(3){(left, right) => left * 41 + right}
- }
-
- /** Element wise update function
- *
- * @param row row index
- * @param col column index
- * @param value value to set at (row, col)
- */
- override def update(row: Int, col: Int, value: Double): Unit = {
- val index = locate(row, col)
-
- data(index) = value
- }
-
- /** Converts the DenseMatrix to a SparseMatrix
- *
- * @return SparseMatrix build from all the non-null values
- */
- def toSparseMatrix: SparseMatrix = {
- val entries = for(row <- 0 until numRows; col <- 0 until numCols) yield {
- (row, col, apply(row, col))
- }
-
- SparseMatrix.fromCOO(numRows, numCols, entries.filter(_._3 != 0))
- }
-
- /** Calculates the linear index of the respective matrix entry
- *
- * @param row row index
- * @param col column index
- * @return the index of the value according to the row and index
- */
- private def locate(row: Int, col: Int): Int = {
- require(0 <= row && row < numRows && 0 <= col && col < numCols,
- (row, col) + " not in [0, " + numRows + ") x [0, " + numCols + ")")
-
- row + col * numRows
- }
-
- /** Converts the entry at (row, col) to string
- *
- * @param row row index
- * @param col column index
- * @return Takes the value according to the row and index and convert it to string
- */
- private def printEntry(row: Int, col: Int): String = {
- val index = locate(row, col)
-
- data(index).toString
- }
-
- /** Copies the matrix instance
- *
- * @return Copy of itself
- */
- override def copy: DenseMatrix = {
- new DenseMatrix(numRows, numCols, data.clone)
- }
-}
-
-object DenseMatrix {
-
- val LINE_WIDTH = 100
- val MAX_ROWS = 50
-
- def apply(numRows: Int, numCols: Int, values: Array[Int]): DenseMatrix = {
- new DenseMatrix(numRows, numCols, values.map(_.toDouble))
- }
-
- def apply(numRows: Int, numCols: Int, values: Double*): DenseMatrix = {
- new DenseMatrix(numRows, numCols, values.toArray)
- }
-
- def zeros(numRows: Int, numCols: Int): DenseMatrix = {
- new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(0.0))
- }
-
- def eye(numRows: Int, numCols: Int): DenseMatrix = {
- new DenseMatrix(numRows, numCols, Array.fill(numRows * numCols)(1.0))
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/DenseVector.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/DenseVector.scala
deleted file mode 100644
index c0c37bb..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/DenseVector.scala
+++ /dev/null
@@ -1,187 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import breeze.linalg.{SparseVector => BreezeSparseVector, DenseVector => BreezeDenseVector, Vector => BreezeVector}
-
-/**
- * Dense vector implementation of [[Vector]]. The data is represented in a continuous array of
- * doubles.
- *
- * @param data Array of doubles to store the vector elements
- */
-case class DenseVector(data: Array[Double]) extends Vector with Serializable {
-
- /**
- * Number of elements in a vector
- * @return the number of the elements in the vector
- */
- override def size: Int = {
- data.length
- }
-
- /**
- * Element wise access function
- *
- * @param index index of the accessed element
- * @return element at the given index
- */
- override def apply(index: Int): Double = {
- require(0 <= index && index < data.length, index + " not in [0, " + data.length + ")")
- data(index)
- }
-
- override def toString: String = {
- s"DenseVector(${data.mkString(", ")})"
- }
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case dense: DenseVector => data.length == dense.data.length && data.sameElements(dense.data)
- case _ => false
- }
- }
-
- override def hashCode: Int = {
- java.util.Arrays.hashCode(data)
- }
-
- /**
- * Copies the vector instance
- *
- * @return Copy of the vector instance
- */
- override def copy: DenseVector = {
- DenseVector(data.clone())
- }
-
- /** Updates the element at the given index with the provided value
- *
- * @param index Index whose value is updated.
- * @param value The value used to update the index.
- */
- override def update(index: Int, value: Double): Unit = {
- require(0 <= index && index < data.length, index + " not in [0, " + data.length + ")")
-
- data(index) = value
- }
-
- /** Returns the dot product of the recipient and the argument
- *
- * @param other a Vector
- * @return a scalar double of dot product
- */
- override def dot(other: Vector): Double = {
- require(size == other.size, "The size of vector must be equal.")
-
- other match {
- case SparseVector(_, otherIndices, otherData) =>
- otherIndices.zipWithIndex.map {
- case (idx, sparseIdx) => data(idx) * otherData(sparseIdx)
- }.sum
- case _ => (0 until size).map(i => data(i) * other(i)).sum
- }
- }
-
- /** Returns the outer product (a.k.a. Kronecker product) of `this`
- * with `other`. The result will given in [[org.apache.flink.ml.math.SparseMatrix]]
- * representation if `other` is sparse and as [[org.apache.flink.ml.math.DenseMatrix]] otherwise.
- *
- * @param other a Vector
- * @return the [[org.apache.flink.ml.math.Matrix]] which equals the outer product of `this`
- * with `other.`
- */
- override def outer(other: Vector): Matrix = {
- val numRows = size
- val numCols = other.size
-
- other match {
- case sv: SparseVector =>
- val entries = for {
- i <- 0 until numRows
- (j, k) <- sv.indices.zipWithIndex
- value = this(i) * sv.data(k)
- if value != 0
- } yield (i, j, value)
-
- SparseMatrix.fromCOO(numRows, numCols, entries)
- case _ =>
- val values = for {
- i <- 0 until numRows
- j <- 0 until numCols
- } yield this(i) * other(j)
-
- DenseMatrix(numRows, numCols, values.toArray)
- }
- }
-
- /** Magnitude of a vector
- *
- * @return The length of the vector
- */
- override def magnitude: Double = {
- math.sqrt(data.map(x => x * x).sum)
- }
-
- /** Convert to a [[SparseVector]]
- *
- * @return Creates a SparseVector from the DenseVector
- */
- def toSparseVector: SparseVector = {
- val nonZero = (0 until size).zip(data).filter(_._2 != 0)
-
- SparseVector.fromCOO(size, nonZero)
- }
-}
-
-object DenseVector {
-
- def apply(values: Double*): DenseVector = {
- new DenseVector(values.toArray)
- }
-
- def apply(values: Array[Int]): DenseVector = {
- new DenseVector(values.map(_.toDouble))
- }
-
- def zeros(size: Int): DenseVector = {
- init(size, 0.0)
- }
-
- def eye(size: Int): DenseVector = {
- init(size, 1.0)
- }
-
- def init(size: Int, value: Double): DenseVector = {
- new DenseVector(Array.fill(size)(value))
- }
-
- /** BreezeVectorConverter implementation for [[org.apache.flink.ml.math.DenseVector]]
- *
- * This allows to convert Breeze vectors into [[DenseVector]].
- */
- implicit val denseVectorConverter = new BreezeVectorConverter[DenseVector] {
- override def convert(vector: BreezeVector[Double]): DenseVector = {
- vector match {
- case dense: BreezeDenseVector[Double] => new DenseVector(dense.data)
- case sparse: BreezeSparseVector[Double] => new DenseVector(sparse.toDenseVector.data)
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Matrix.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Matrix.scala
deleted file mode 100644
index ea2b4cd..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Matrix.scala
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-/** Base trait for a matrix representation
- *
- */
-trait Matrix {
-
- /** Number of rows
- *
- * @return number of rows in the matrix
- */
- def numRows: Int
-
- /** Number of columns
- *
- * @return number of columns in the matrix
- */
- def numCols: Int
-
- /** Element wise access function
- *
- * @param row row index
- * @param col column index
- * @return matrix entry at (row, col)
- */
- def apply(row: Int, col: Int): Double
-
- /** Element wise update function
- *
- * @param row row index
- * @param col column index
- * @param value value to set at (row, col)
- */
- def update(row: Int, col: Int, value: Double): Unit
-
- /** Copies the matrix instance
- *
- * @return Copy of itself
- */
- def copy: Matrix
-
- def equalsMatrix(matrix: Matrix): Boolean = {
- if(numRows == matrix.numRows && numCols == matrix.numCols) {
- val coordinates = for(row <- 0 until numRows; col <- 0 until numCols) yield (row, col)
- coordinates forall { case(row, col) => this.apply(row, col) == matrix(row, col)}
- } else {
- false
- }
- }
-
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/SparseMatrix.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/SparseMatrix.scala
deleted file mode 100644
index d7dd9dc..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/SparseMatrix.scala
+++ /dev/null
@@ -1,267 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import scala.util.Sorting
-
-/** Sparse matrix using the compressed sparse column (CSC) representation.
- *
- * More details concerning the compressed sparse column (CSC) representation can be found
- * [http://en.wikipedia.org/wiki/Sparse_matrix#Compressed_sparse_column_.28CSC_or_CCS.29].
- *
- * @param numRows Number of rows
- * @param numCols Number of columns
- * @param rowIndices Array containing the row indices of non-zero entries
- * @param colPtrs Array containing the starting offsets in data for each column
- * @param data Array containing the non-zero entries in column-major order
- */
-class SparseMatrix(
- val numRows: Int,
- val numCols: Int,
- val rowIndices: Array[Int],
- val colPtrs: Array[Int],
- val data: Array[Double])
- extends Matrix
- with Serializable {
-
- /** Element wise access function
- *
- * @param row row index
- * @param col column index
- * @return matrix entry at (row, col)
- */
- override def apply(row: Int, col: Int): Double = {
- val index = locate(row, col)
-
- if(index < 0){
- 0
- } else {
- data(index)
- }
- }
-
- def toDenseMatrix: DenseMatrix = {
- val result = DenseMatrix.zeros(numRows, numCols)
-
- for(row <- 0 until numRows; col <- 0 until numCols) {
- result(row, col) = apply(row, col)
- }
-
- result
- }
-
- /** Element wise update function
- *
- * @param row row index
- * @param col column index
- * @param value value to set at (row, col)
- */
- override def update(row: Int, col: Int, value: Double): Unit = {
- val index = locate(row, col)
-
- if(index < 0) {
- throw new IllegalArgumentException("Cannot update zero value of sparse matrix at index " +
- s"($row, $col)")
- } else {
- data(index) = value
- }
- }
-
- override def toString: String = {
- val result = StringBuilder.newBuilder
-
- result.append(s"SparseMatrix($numRows, $numCols)\n")
-
- var columnIndex = 0
-
- val fieldWidth = math.max(numRows, numCols).toString.length
- val valueFieldWidth = data.map(_.toString.length).max + 2
-
- for(index <- 0 until colPtrs.last) {
- while(colPtrs(columnIndex + 1) <= index){
- columnIndex += 1
- }
-
- val rowStr = rowIndices(index).toString
- val columnStr = columnIndex.toString
- val valueStr = data(index).toString
-
- result.append("(" + " " * (fieldWidth - rowStr.length) + rowStr + "," +
- " " * (fieldWidth - columnStr.length) + columnStr + ")")
- result.append(" " * (valueFieldWidth - valueStr.length) + valueStr)
- result.append("\n")
- }
-
- result.toString
- }
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case sm: SparseMatrix if numRows == sm.numRows && numCols == sm.numCols =>
- rowIndices.sameElements(sm.rowIndices) && colPtrs.sameElements(sm.colPtrs) &&
- data.sameElements(sm.data)
- case _ => false
- }
- }
-
- override def hashCode: Int = {
- val hashCodes = List(numRows.hashCode(), numCols.hashCode(),
- java.util.Arrays.hashCode(rowIndices), java.util.Arrays.hashCode(colPtrs),
- java.util.Arrays.hashCode(data))
-
- hashCodes.foldLeft(5){(left, right) => left * 41 + right}
- }
-
- private def locate(row: Int, col: Int): Int = {
- require(0 <= row && row < numRows && 0 <= col && col < numCols,
- (row, col) + " not in [0, " + numRows + ") x [0, " + numCols + ")")
-
- val startIndex = colPtrs(col)
- val endIndex = colPtrs(col + 1)
-
- java.util.Arrays.binarySearch(rowIndices, startIndex, endIndex, row)
- }
-
- /** Copies the matrix instance
- *
- * @return Copy of itself
- */
- override def copy: SparseMatrix = {
- new SparseMatrix(numRows, numCols, rowIndices.clone, colPtrs.clone(), data.clone)
- }
-}
-
-object SparseMatrix{
-
- /** Constructs a sparse matrix from a coordinate list (COO) representation where each entry
- * is stored as a tuple of (rowIndex, columnIndex, value).
- *
- * @param numRows Number of rows
- * @param numCols Number of columns
- * @param entries Data entries in the matrix
- * @return Newly constructed sparse matrix
- */
- def fromCOO(numRows: Int, numCols: Int, entries: (Int, Int, Double)*): SparseMatrix = {
- fromCOO(numRows, numCols, entries)
- }
-
- /** Constructs a sparse matrix from a coordinate list (COO) representation where each entry
- * is stored as a tuple of (rowIndex, columnIndex, value).
- *
- * @param numRows Number of rows
- * @param numCols Number of columns
- * @param entries Data entries in the matrix
- * @return Newly constructed sparse matrix
- */
- def fromCOO(numRows: Int, numCols: Int, entries: Iterable[(Int, Int, Double)]): SparseMatrix = {
- val entryArray = entries.toArray
-
- entryArray.foreach{ case (row, col, _) =>
- require(0 <= row && row < numRows && 0 <= col && col <= numCols,
- (row, col) + " not in [0, " + numRows + ") x [0, " + numCols + ")")
- }
-
- val COOOrdering = new Ordering[(Int, Int, Double)] {
- override def compare(x: (Int, Int, Double), y: (Int, Int, Double)): Int = {
- if(x._2 < y._2) {
- -1
- } else if(x._2 > y._2) {
- 1
- } else {
- x._1 - y._1
- }
- }
- }
-
- Sorting.quickSort(entryArray)(COOOrdering)
-
- val nnz = entryArray.length
-
- val data = new Array[Double](nnz)
- val rowIndices = new Array[Int](nnz)
- val colPtrs = new Array[Int](numCols + 1)
-
- var (lastRow, lastCol, lastValue) = entryArray(0)
-
- rowIndices(0) = lastRow
- data(0) = lastValue
-
- var i = 1
- var lastDataIndex = 0
-
- while(i < nnz) {
- val (curRow, curCol, curValue) = entryArray(i)
-
- if(lastRow == curRow && lastCol == curCol) {
- // add values with identical coordinates
- data(lastDataIndex) += curValue
- } else {
- lastDataIndex += 1
- data(lastDataIndex) = curValue
- rowIndices(lastDataIndex) = curRow
- lastRow = curRow
- }
-
- while(lastCol < curCol) {
- lastCol += 1
- colPtrs(lastCol) = lastDataIndex
- }
-
- i += 1
- }
-
- lastDataIndex += 1
- while(lastCol < numCols) {
- colPtrs(lastCol + 1) = lastDataIndex
- lastCol += 1
- }
-
- val prunedRowIndices = if(lastDataIndex < nnz) {
- val prunedArray = new Array[Int](lastDataIndex)
- rowIndices.copyToArray(prunedArray)
- prunedArray
- } else {
- rowIndices
- }
-
- val prunedData = if(lastDataIndex < nnz) {
- val prunedArray = new Array[Double](lastDataIndex)
- data.copyToArray(prunedArray)
- prunedArray
- } else {
- data
- }
-
- new SparseMatrix(numRows, numCols, prunedRowIndices, colPtrs, prunedData)
- }
-
- /** Convenience method to convert a single tuple with an integer value into a SparseMatrix.
- * The problem is that providing a single tuple to the fromCOO method, the Scala type inference
- * cannot infer that the tuple has to be of type (Int, Int, Double) because of the overloading
- * with the Iterable type.
- *
- * @param numRows Number of rows
- * @param numCols Number of columns
- * @param entry Data entries in the matrix
- * @return Newly constructed sparse matrix
- */
- def fromCOO(numRows: Int, numCols: Int, entry: (Int, Int, Int)): SparseMatrix = {
- fromCOO(numRows, numCols, (entry._1, entry._2, entry._3.toDouble))
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/SparseVector.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/SparseVector.scala
deleted file mode 100644
index 29c16cc..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/SparseVector.scala
+++ /dev/null
@@ -1,283 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import breeze.linalg.{SparseVector => BreezeSparseVector, DenseVector => BreezeDenseVector, Vector => BreezeVector}
-
-import scala.util.Sorting
-
-/** Sparse vector implementation storing the data in two arrays. One index contains the sorted
- * indices of the non-zero vector entries and the other the corresponding vector entries
- */
-case class SparseVector(size: Int, indices: Array[Int], data: Array[Double])
- extends Vector with Serializable {
-
- /** Updates the element at the given index with the provided value
- *
- * @param index Index whose value is updated.
- * @param value The value used to update the index.
- */
- override def update(index: Int, value: Double): Unit = {
- val resolvedIndex = locate(index)
-
- if (resolvedIndex < 0) {
- throw new IllegalArgumentException("Cannot update zero value of sparse vector at " +
- s"index $index")
- } else {
- data(resolvedIndex) = value
- }
- }
-
- /** Copies the vector instance
- *
- * @return Copy of the [[SparseVector]] instance
- */
- override def copy: SparseVector = {
- new SparseVector(size, indices.clone, data.clone)
- }
-
- /** Returns the dot product of the recipient and the argument
- *
- * @param other a Vector
- * @return a scalar double of dot product
- */
- override def dot(other: Vector): Double = {
- require(size == other.size, "The size of vector must be equal.")
- other match {
- case DenseVector(otherData) =>
- indices.zipWithIndex.map { case (sparseIdx, idx) => data(idx) * otherData(sparseIdx) }.sum
- case SparseVector(_, otherIndices, otherData) =>
- var left = 0
- var right = 0
- var result = 0.0
-
- while (left < indices.length && right < otherIndices.length) {
- if (indices(left) < otherIndices(right)) {
- left += 1
- } else if (otherIndices(right) < indices(left)) {
- right += 1
- } else {
- result += data(left) * otherData(right)
- left += 1
- right += 1
- }
- }
- result
- }
- }
-
- /** Returns the outer product (a.k.a. Kronecker product) of `this` with `other`. The result is
- * given in [[SparseMatrix]] representation.
- *
- * @param other a [[Vector]]
- * @return the [[SparseMatrix]] which equals the outer product of `this` with `other.`
- */
- override def outer(other: Vector): SparseMatrix = {
- val numRows = size
- val numCols = other.size
-
- val entries = other match {
- case sv: SparseVector =>
- for {
- (i, k) <- indices.zipWithIndex
- (j, l) <- sv.indices.zipWithIndex
- value = data(k) * sv.data(l)
- if value != 0
- } yield (i, j, value)
- case _ =>
- for {
- (i, k) <- indices.zipWithIndex
- j <- 0 until numCols
- value = data(k) * other(j)
- if value != 0
- } yield (i, j, value)
- }
-
- SparseMatrix.fromCOO(numRows, numCols, entries)
- }
-
-
- /** Magnitude of a vector
- *
- * @return The length of the vector
- */
- override def magnitude: Double = math.sqrt(data.map(x => x * x).sum)
-
- /** Element wise access function
- *
- * * @param index index of the accessed element
- * @return element with index
- */
- override def apply(index: Int): Double = {
- val resolvedIndex = locate(index)
-
- if(resolvedIndex < 0) {
- 0
- } else {
- data(resolvedIndex)
- }
- }
-
- /** Converts the [[SparseVector]] to a [[DenseVector]]
- *
- * @return The DenseVector out of the SparseVector
- */
- def toDenseVector: DenseVector = {
- val denseVector = DenseVector.zeros(size)
-
- for(index <- 0 until size) {
- denseVector(index) = this(index)
- }
-
- denseVector
- }
-
- override def equals(obj: Any): Boolean = {
- obj match {
- case sv: SparseVector if size == sv.size =>
- indices.sameElements(sv.indices) && data.sameElements(sv.data)
- case _ => false
- }
- }
-
- override def hashCode: Int = {
- val hashCodes = List(size.hashCode, java.util.Arrays.hashCode(indices),
- java.util.Arrays.hashCode(data))
-
- hashCodes.foldLeft(3){ (left, right) => left * 41 + right}
- }
-
- override def toString: String = {
- val entries = indices.zip(data).mkString(", ")
- "SparseVector(" + entries + ")"
- }
-
- private def locate(index: Int): Int = {
- require(0 <= index && index < size, index + " not in [0, " + size + ")")
-
- java.util.Arrays.binarySearch(indices, 0, indices.length, index)
- }
-}
-
-object SparseVector {
-
- /** Constructs a sparse vector from a coordinate list (COO) representation where each entry
- * is stored as a tuple of (index, value).
- *
- * @param size The number of elements in the vector
- * @param entries The values in the vector
- * @return a new [[SparseVector]]
- */
- def fromCOO(size: Int, entries: (Int, Double)*): SparseVector = {
- fromCOO(size, entries)
- }
-
- /** Constructs a sparse vector from a coordinate list (COO) representation where each entry
- * is stored as a tuple of (index, value).
- *
- * @param size The number of elements in the vector
- * @param entries An iterator supplying the values in the vector
- * @return a new [[SparseVector]]
- */
- def fromCOO(size: Int, entries: Iterable[(Int, Double)]): SparseVector = {
- val entryArray = entries.toArray
-
- entryArray.foreach { case (index, _) =>
- require(0 <= index && index < size, index + " not in [0, " + size + ")")
- }
-
- val COOOrdering = new Ordering[(Int, Double)] {
- override def compare(x: (Int, Double), y: (Int, Double)): Int = {
- x._1 - y._1
- }
- }
-
- Sorting.quickSort(entryArray)(COOOrdering)
-
- // calculate size of the array
- val arraySize = entryArray.foldLeft((-1, 0)){ case ((lastIndex, numRows), (index, _)) =>
- if(lastIndex == index) {
- (lastIndex, numRows)
- } else {
- (index, numRows + 1)
- }
- }._2
-
- val indices = new Array[Int](arraySize)
- val data = new Array[Double](arraySize)
-
- val (index, value) = entryArray(0)
-
- indices(0) = index
- data(0) = value
-
- var i = 1
- var lastIndex = indices(0)
- var lastDataIndex = 0
-
- while(i < entryArray.length) {
- val (curIndex, curValue) = entryArray(i)
-
- if(curIndex == lastIndex) {
- data(lastDataIndex) += curValue
- } else {
- lastDataIndex += 1
- data(lastDataIndex) = curValue
- indices(lastDataIndex) = curIndex
- lastIndex = curIndex
- }
-
- i += 1
- }
-
- new SparseVector(size, indices, data)
- }
-
- /** Convenience method to be able to instantiate a SparseVector with a single element. The Scala
- * type inference mechanism cannot infer that the second tuple value has to be of type Double
- * if only a single tuple is provided.
- *
- * @param size The number of elements in the vector
- * @param entry The value in the vector
- * @return a new [[SparseVector]]
- */
- def fromCOO(size: Int, entry: (Int, Int)): SparseVector = {
- fromCOO(size, (entry._1, entry._2.toDouble))
- }
-
- /** BreezeVectorConverter implementation for [[org.apache.flink.ml.math.SparseVector]]
- *
- * This allows to convert Breeze vectors into [[SparseVector]]
- */
- implicit val sparseVectorConverter = new BreezeVectorConverter[SparseVector] {
- override def convert(vector: BreezeVector[Double]): SparseVector = {
- vector match {
- case dense: BreezeDenseVector[Double] =>
- SparseVector.fromCOO(
- dense.length,
- dense.iterator.toIterable)
- case sparse: BreezeSparseVector[Double] =>
- new SparseVector(
- sparse.length,
- sparse.index.take(sparse.used),
- sparse.data.take(sparse.used))
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Vector.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Vector.scala
deleted file mode 100644
index 0c911d0..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/Vector.scala
+++ /dev/null
@@ -1,103 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-import breeze.linalg.{SparseVector => BreezeSparseVector, DenseVector => BreezeDenseVector, Vector => BreezeVector}
-
-/** Base trait for Vectors
- *
- */
-trait Vector extends Serializable {
-
- /** Number of elements in a vector
- *
- * @return The number of elements of the vector
- */
- def size: Int
-
- /** Element wise access function
- *
- * @param index index of the accessed element
- * @return value of the associated with the index
- */
- def apply(index: Int): Double
-
- /** Updates the element at the given index with the provided value
- *
- * @param index The index of the element to be updated
- * @param value The new value
- */
- def update(index: Int, value: Double): Unit
-
- /** Copies the vector instance
- *
- * @return Copy of the vector instance
- */
- def copy: Vector
-
- /** Returns the dot product of the recipient and the argument
- *
- * @param other a Vector
- * @return a scalar double of dot product
- */
- def dot(other: Vector): Double
-
- /** Returns the outer product of the recipient and the argument
- *
- * @param other a Vector
- * @return a matrix
- */
- def outer(other: Vector): Matrix
-
- /** Magnitude of a vector
- *
- * @return The length of the vector
- */
- def magnitude: Double
-
- def equalsVector(vector: Vector): Boolean = {
- if(size == vector.size) {
- (0 until size) forall { idx =>
- this(idx) == vector(idx)
- }
- } else {
- false
- }
- }
-}
-
-object Vector{
- /** BreezeVectorConverter implementation for [[Vector]]
- *
- * This allows to convert Breeze vectors into [[Vector]].
- */
- implicit val vectorConverter = new BreezeVectorConverter[Vector] {
- override def convert(vector: BreezeVector[Double]): Vector = {
- vector match {
- case dense: BreezeDenseVector[Double] => new DenseVector(dense.data)
-
- case sparse: BreezeSparseVector[Double] =>
- new SparseVector(
- sparse.length,
- sparse.index.take(sparse.used),
- sparse.data.take(sparse.used))
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/VectorBuilder.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/VectorBuilder.scala
deleted file mode 100644
index 3bbf146..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/VectorBuilder.scala
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math
-
-/** Type class to allow the vector construction from different data types
- *
- * @tparam T Subtype of [[Vector]]
- */
-trait VectorBuilder[T <: Vector] extends Serializable {
- /** Builds a [[Vector]] of type T from a List[Double]
- *
- * @param data Input data where the index denotes the resulting index of the vector
- * @return A vector of type T
- */
- def build(data: List[Double]): T
-}
-
-object VectorBuilder{
-
- /** Type class implementation for [[org.apache.flink.ml.math.DenseVector]] */
- implicit val denseVectorBuilder = new VectorBuilder[DenseVector] {
- override def build(data: List[Double]): DenseVector = {
- new DenseVector(data.toArray)
- }
- }
-
- /** Type class implementation for [[org.apache.flink.ml.math.SparseVector]] */
- implicit val sparseVectorBuilder = new VectorBuilder[SparseVector] {
- override def build(data: List[Double]): SparseVector = {
- // Enrich elements with explicit indices and filter out zero entries
- SparseVector.fromCOO(data.length, data.indices.zip(data).filter(_._2 != 0.0))
- }
- }
-
- /** Type class implementation for [[Vector]] */
- implicit val vectorBuilder = new VectorBuilder[Vector] {
- override def build(data: List[Double]): Vector = {
- new DenseVector(data.toArray)
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/distributed/DistributedMatrix.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/distributed/DistributedMatrix.scala
deleted file mode 100644
index f34641e..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/distributed/DistributedMatrix.scala
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math.distributed
-
-/** Common trait used by distributed data structures representing a matrix. */
-trait DistributedMatrix {
- /** Returns number of rows in matrix.
- *
- * @return Number of rows
- */
- def numRows: Int
-
- /** Returns number of columns in matrix.
- *
- * @return Number of columns
- */
- def numCols: Int
-}
-
-object DistributedMatrix {
- type MatrixColIndex = Int
- type MatrixRowIndex = Int
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrix.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrix.scala
deleted file mode 100644
index 5dca01a..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/distributed/DistributedRowMatrix.scala
+++ /dev/null
@@ -1,172 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.math.distributed
-
-import org.apache.flink.api.scala._
-import org.apache.flink.ml.math._
-import org.apache.flink.ml.math.Breeze._
-import org.apache.flink.ml.math.distributed.DistributedMatrix._
-
-/** Represents distributed row-major matrix.
- *
- * @param data [[DataSet]] which contains [[IndexedRow]]s
- * @param numRows Number of rows
- * @param numCols Number of columns
- */
-class DistributedRowMatrix(
- val data: DataSet[IndexedRow],
- val numRows: Int,
- val numCols: Int
-) extends DistributedMatrix {
-
- /** Collects the data in the form of a sequence of coordinates associated with their values.
- * This operation immediately triggers program execution.
- *
- * @return Returns the matrix in the sparse coordinate format
- */
- def toCOO: Seq[(MatrixRowIndex, MatrixColIndex, Double)] = {
- val localRows = data.collect()
-
- for {
- IndexedRow(rowIndex, vector) <- localRows
- (columnIndex, value) <- vector
- } yield (rowIndex, columnIndex, value)
- }
-
- /** Collects the data in the form of a SparseMatrix. This operation immediately triggers program
- * execution.
- *
- * @return Returns the matrix as a local [[SparseMatrix]]
- */
- def toLocalSparseMatrix: SparseMatrix = {
- val localMatrix = SparseMatrix.fromCOO(this.numRows, this.numCols, this.toCOO)
- require(localMatrix.numRows == this.numRows)
- require(localMatrix.numCols == this.numCols)
-
- localMatrix
- }
-
- // TODO: convert to dense representation on the distributed matrix and collect it afterward
- /** Collects the data in the form of a DenseMatrix. This operation immediately triggers program
- * execution.
- *
- * @return Returns the matrix as a [[DenseMatrix]]
- */
- def toLocalDenseMatrix: DenseMatrix = this.toLocalSparseMatrix.toDenseMatrix
-
- /** Applies a high-order function to couple of rows.
- *
- * @param func a function to be applied
- * @param other a [[DistributedRowMatrix]] to apply the function together
- * @return Applies the function and returns a new [[DistributedRowMatrix]]
- */
- def byRowOperation(
- func: (Vector, Vector) => Vector,
- other: DistributedRowMatrix
- ): DistributedRowMatrix = {
- val otherData = other.data
- require(this.numCols == other.numCols)
- require(this.numRows == other.numRows)
-
- val result = this.data
- .fullOuterJoin(otherData)
- .where("rowIndex")
- .equalTo("rowIndex")(
- (left: IndexedRow, right: IndexedRow) => {
- val row1 = Option(left) match {
- case Some(row: IndexedRow) => row
- case None =>
- IndexedRow(right.rowIndex, SparseVector.fromCOO(right.values.size, List((0, 0.0))))
- }
- val row2 = Option(right) match {
- case Some(row: IndexedRow) => row
- case None =>
- IndexedRow(left.rowIndex, SparseVector.fromCOO(left.values.size, List((0, 0.0))))
- }
- IndexedRow(row1.rowIndex, func(row1.values, row2.values))
- }
- )
- new DistributedRowMatrix(result, numRows, numCols)
- }
-
- /** Adds this matrix to another matrix.
- *
- * @param other a [[DistributedRowMatrix]] to be added
- * @return [[DistributedRowMatrix]] representing the two matrices added.
- */
- def add(other: DistributedRowMatrix): DistributedRowMatrix = {
- val addFunction = (x: Vector, y: Vector) => (x.asBreeze + y.asBreeze).fromBreeze
- this.byRowOperation(addFunction, other)
- }
-
- /** Subtracts another matrix from this matrix.
- *
- * @param other a [[DistributedRowMatrix]] to be subtracted from this matrix
- * @return [[DistributedRowMatrix]] representing the original matrix subtracted by the supplied
- * matrix.
- */
- def subtract(other: DistributedRowMatrix): DistributedRowMatrix = {
- val subFunction = (x: Vector, y: Vector) => (x.asBreeze - y.asBreeze).fromBreeze
- this.byRowOperation(subFunction, other)
- }
-}
-
-object DistributedRowMatrix {
-
- /** Builds a [[DistributedRowMatrix]] from a [[DataSet]] in COO.
- *
- * @param data [[DataSet]] which contains matrix elements in the form of
- * (row index, column index, value)
- * @param numRows Number of rows
- * @param numCols Number of columns
- * @param isSorted If false, sorts the row to properly build the matrix representation.
- * If already sorted, set this parameter to true to skip sorting.
- * @return the [[DistributedRowMatrix]] build from the original coordinate matrix
- */
- def fromCOO(data: DataSet[(MatrixRowIndex, MatrixColIndex, Double)],
- numRows: Int,
- numCols: Int,
- isSorted: Boolean = false
- ): DistributedRowMatrix = {
- val vectorData: DataSet[(MatrixRowIndex, SparseVector)] = data
- .groupBy(0)
- .reduceGroup(sparseRow => {
- require(sparseRow.nonEmpty)
- val sortedRow =
- if (isSorted) {
- sparseRow.toList
- } else {
- sparseRow.toList.sortBy(row => row._2)
- }
- val (indices, values) = sortedRow.map(x => (x._2, x._3)).unzip
- (sortedRow.head._1, SparseVector(numCols, indices.toArray, values.toArray))
- })
-
- val zippedData = vectorData.map(x => IndexedRow(x._1.toInt, x._2))
-
- new DistributedRowMatrix(zippedData, numRows, numCols)
- }
-}
-
-/** Represents a row in row-major matrix. */
-case class IndexedRow(rowIndex: MatrixRowIndex, values: Vector) extends Ordered[IndexedRow] {
- def compare(other: IndexedRow) = this.rowIndex.compare(other.rowIndex)
-
- override def toString: String = s"($rowIndex, ${values.toString})"
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/package.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/package.scala
deleted file mode 100644
index 7493602..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/math/package.scala
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml
-
-/**
- * Convenience methods to handle Flink's [[org.apache.flink.ml.math.Matrix]] and [[Vector]]
- * abstraction.
- */
-package object math {
- implicit class RichMatrix(matrix: Matrix) extends Iterable[(Int, Int, Double)] {
-
- override def iterator: Iterator[(Int, Int, Double)] = {
- new Iterator[(Int, Int, Double)] {
- var index = 0
-
- override def hasNext: Boolean = {
- index < matrix.numRows * matrix.numCols
- }
-
- override def next(): (Int, Int, Double) = {
- val row = index % matrix.numRows
- val column = index / matrix.numRows
-
- index += 1
-
- (row, column, matrix(row, column))
- }
- }
- }
-
- def valueIterator: Iterator[Double] = {
- val it = iterator
-
- new Iterator[Double] {
- override def hasNext: Boolean = it.hasNext
-
- override def next(): Double = it.next._3
- }
- }
-
- }
-
- implicit class RichVector(vector: Vector) extends Iterable[(Int, Double)] {
-
- override def iterator: Iterator[(Int, Double)] = {
- new Iterator[(Int, Double)] {
- var index = 0
-
- override def hasNext: Boolean = {
- index < vector.size
- }
-
- override def next(): (Int, Double) = {
- val resultIndex = index
-
- index += 1
-
- (resultIndex, vector(resultIndex))
- }
- }
- }
-
- def valueIterator: Iterator[Double] = {
- val it = iterator
-
- new Iterator[Double] {
- override def hasNext: Boolean = it.hasNext
-
- override def next(): Double = it.next._2
- }
- }
- }
-
- /** Stores the vector values in a dense array
- *
- * @param vector Subtype of [[Vector]]
- * @return Array containing the vector values
- */
- def vector2Array(vector: Vector): Array[Double] = {
- vector match {
- case dense: DenseVector => dense.data.clone
-
- case sparse: SparseVector => {
- val result = new Array[Double](sparse.size)
-
- for ((index, value) <- sparse) {
- result(index) = value
- }
-
- result
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/ChebyshevDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/ChebyshevDistanceMetric.scala
deleted file mode 100644
index 055ede3..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/ChebyshevDistanceMetric.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class implements a Chebyshev distance metric. The class calculates the distance between
- * the given vectors by finding the maximum difference between each coordinate.
- *
- * @see http://en.wikipedia.org/wiki/Chebyshev_distance
- */
-class ChebyshevDistanceMetric extends DistanceMetric {
- override def distance(a: Vector, b: Vector): Double = {
- checkValidArguments(a, b)
- (0 until a.size).map(i => math.abs(a(i) - b(i))).max
- }
-}
-
-object ChebyshevDistanceMetric {
- def apply() = new ChebyshevDistanceMetric()
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/CosineDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/CosineDistanceMetric.scala
deleted file mode 100644
index f32ea26..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/CosineDistanceMetric.scala
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class implements a cosine distance metric. The class calculates the distance between
- * the given vectors by dividing the dot product of two vectors by the product of their lengths.
- * We convert the result of division to a usable distance. So, 1 - cos(angle) is actually returned.
- *
- * @see http://en.wikipedia.org/wiki/Cosine_similarity
- */
-class CosineDistanceMetric extends DistanceMetric {
- override def distance(a: Vector, b: Vector): Double = {
- checkValidArguments(a, b)
-
- val dotProd: Double = a.dot(b)
- val denominator: Double = a.magnitude * b.magnitude
- if (dotProd == 0 && denominator == 0) {
- 0
- } else {
- 1 - dotProd / denominator
- }
- }
-}
-
-object CosineDistanceMetric {
- def apply() = new CosineDistanceMetric()
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/DistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/DistanceMetric.scala
deleted file mode 100644
index 21573fe..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/DistanceMetric.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** DistanceMeasure interface is used for object which determines distance between two points.
- */
-trait DistanceMetric extends Serializable {
- /** Returns the distance between the arguments.
- *
- * @param a a Vector defining a multi-dimensional point in some space
- * @param b a Vector defining a multi-dimensional point in some space
- * @return a scalar double of the distance
- */
- def distance(a: Vector, b: Vector): Double
-
- protected def checkValidArguments(a: Vector, b: Vector) = {
- require(a.size == b.size, "The each size of vectors must be same to calculate distance.")
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/EuclideanDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/EuclideanDistanceMetric.scala
deleted file mode 100644
index 153fb93..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/EuclideanDistanceMetric.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class implements a Euclidean distance metric. The metric calculates the distance between
- * the given two vectors by summing the square root of the squared differences between
- * each coordinate.
- *
- * http://en.wikipedia.org/wiki/Euclidean_distance
- *
- * If you don't care about the true distance and only need for comparison,
- * [[SquaredEuclideanDistanceMetric]] will be faster because it doesn't calculate the actual
- * square root of the distances.
- *
- * @see http://en.wikipedia.org/wiki/Euclidean_distance
- */
-class EuclideanDistanceMetric extends SquaredEuclideanDistanceMetric {
- override def distance(a: Vector, b: Vector): Double = math.sqrt(super.distance(a, b))
-}
-
-object EuclideanDistanceMetric {
- def apply() = new EuclideanDistanceMetric()
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/ManhattanDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/ManhattanDistanceMetric.scala
deleted file mode 100644
index 5983f79..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/ManhattanDistanceMetric.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class implements a Manhattan distance metric. The class calculates the distance between
- * the given vectors by summing the differences between each coordinate.
- *
- * @see http://en.wikipedia.org/wiki/Taxicab_geometry
- */
-class ManhattanDistanceMetric extends DistanceMetric{
- override def distance(a: Vector, b: Vector): Double = {
- checkValidArguments(a, b)
- (0 until a.size).map(i => math.abs(a(i) - b(i))).sum
- }
-}
-
-object ManhattanDistanceMetric {
- def apply() = new ManhattanDistanceMetric()
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/MinkowskiDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/MinkowskiDistanceMetric.scala
deleted file mode 100644
index 50161d4..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/MinkowskiDistanceMetric.scala
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class implements a Minkowski distance metric. The metric is a generalization of
- * L(p) distances: Euclidean distance and Manhattan distance. If you need for a special case of
- * p = 1 or p = 2, use [[ManhattanDistanceMetric]], [[EuclideanDistanceMetric]]. This class is
- * useful for high exponents.
- *
- * @param p the norm exponent of space
- *
- * @see http://en.wikipedia.org/wiki/Minkowski_distance
- */
-class MinkowskiDistanceMetric(val p: Double) extends DistanceMetric {
- override def distance(a: Vector, b: Vector): Double = {
- checkValidArguments(a, b)
- math.pow((0 until a.size).map(i => math.pow(math.abs(a(i) - b(i)), p)).sum, 1 / p)
- }
-}
-
-object MinkowskiDistanceMetric {
- def apply(p: Double) = new MinkowskiDistanceMetric(p)
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/SquaredEuclideanDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/SquaredEuclideanDistanceMetric.scala
deleted file mode 100644
index fe546e9..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/SquaredEuclideanDistanceMetric.scala
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class is like [[EuclideanDistanceMetric]] but it does not take the square root.
- *
- * The value calculated by this class is not exact Euclidean distance, but it saves on computation
- * when you need the value for only comparison.
- */
-class SquaredEuclideanDistanceMetric extends DistanceMetric {
- override def distance(a: Vector, b: Vector): Double = {
- checkValidArguments(a, b)
- (0 until a.size).map(i => math.pow(a(i) - b(i), 2)).sum
- }
-}
-
-object SquaredEuclideanDistanceMetric {
- def apply() = new SquaredEuclideanDistanceMetric()
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/TanimotoDistanceMetric.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/TanimotoDistanceMetric.scala
deleted file mode 100644
index 5141c98..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/metrics/distances/TanimotoDistanceMetric.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.metrics.distances
-
-import org.apache.flink.ml.math.Vector
-
-/** This class implements a Tanimoto distance metric. The class calculates the distance between
- * the given vectors. The vectors are assumed as bit-wise vectors. We convert the result of
- * division to a usable distance. So, 1 - similarity is actually returned.
- *
- * @see http://en.wikipedia.org/wiki/Jaccard_index
- */
-class TanimotoDistanceMetric extends DistanceMetric {
- override def distance(a: Vector, b: Vector): Double = {
- checkValidArguments(a, b)
-
- val dotProd: Double = a.dot(b)
- 1 - dotProd / (a.magnitude * a.magnitude + b.magnitude * b.magnitude - dotProd)
- }
-}
-
-object TanimotoDistanceMetric {
- def apply() = new TanimotoDistanceMetric()
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/nn/KNN.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/nn/KNN.scala
deleted file mode 100644
index 6878703..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/nn/KNN.scala
+++ /dev/null
@@ -1,361 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.nn
-
-import org.apache.flink.api.common.operators.Order
-import org.apache.flink.api.common.operators.base.CrossOperatorBase.CrossHint
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.utils._
-import org.apache.flink.ml.common._
-import org.apache.flink.ml.math.{DenseVector, Vector => FlinkVector}
-import org.apache.flink.ml.metrics.distances._
-import org.apache.flink.ml.pipeline.{FitOperation, PredictDataSetOperation, Predictor}
-import org.apache.flink.util.Collector
-
-import scala.collection.immutable.Vector
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-import scala.reflect.ClassTag
-
-/** Implements a `k`-nearest neighbor join.
- *
- * Calculates the `k`-nearest neighbor points in the training set for each point in the test set.
- *
- * @example
- * {{{
- * val trainingDS: DataSet[Vector] = ...
- * val testingDS: DataSet[Vector] = ...
- *
- * val knn = KNN()
- * .setK(10)
- * .setBlocks(5)
- * .setDistanceMetric(EuclideanDistanceMetric())
- *
- * knn.fit(trainingDS)
- *
- * val predictionDS: DataSet[(Vector, Array[Vector])] = knn.predict(testingDS)
- * }}}
- *
- * =Parameters=
- *
- * - [[org.apache.flink.ml.nn.KNN.K]]
- * Sets the K which is the number of selected points as neighbors. (Default value: '''5''')
- *
- * - [[org.apache.flink.ml.nn.KNN.DistanceMetric]]
- * Sets the distance metric we use to calculate the distance between two points. If no metric is
- * specified, then [[org.apache.flink.ml.metrics.distances.EuclideanDistanceMetric]] is used.
- * (Default value: '''EuclideanDistanceMetric()''')
- *
- * - [[org.apache.flink.ml.nn.KNN.Blocks]]
- * Sets the number of blocks into which the input data will be split. This number should be set
- * at least to the degree of parallelism. If no value is specified, then the parallelism of the
- * input [[DataSet]] is used as the number of blocks. (Default value: '''None''')
- *
- * - [[org.apache.flink.ml.nn.KNN.UseQuadTree]]
- * A boolean variable that whether or not to use a quadtree to partition the training set
- * to potentially simplify the KNN search. If no value is specified, the code will
- * automatically decide whether or not to use a quadtree. Use of a quadtree scales well
- * with the number of training and testing points, though poorly with the dimension.
- * (Default value: '''None''')
- *
- * - [[org.apache.flink.ml.nn.KNN.SizeHint]]
- * Specifies whether the training set or test set is small to optimize the cross
- * product operation needed for the KNN search. If the training set is small
- * this should be `CrossHint.FIRST_IS_SMALL` and set to `CrossHint.SECOND_IS_SMALL`
- * if the test set is small.
- * (Default value: '''None''')
- *
- */
-
-class KNN extends Predictor[KNN] {
-
- import KNN._
-
- var trainingSet: Option[DataSet[Block[FlinkVector]]] = None
-
- /** Sets K
- *
- * @param k the number of selected points as neighbors
- */
- def setK(k: Int): KNN = {
- require(k > 0, "K must be positive.")
- parameters.add(K, k)
- this
- }
-
- /** Sets the distance metric
- *
- * @param metric the distance metric to calculate distance between two points
- */
- def setDistanceMetric(metric: DistanceMetric): KNN = {
- parameters.add(DistanceMetric, metric)
- this
- }
-
- /** Sets the number of data blocks/partitions
- *
- * @param n the number of data blocks
- */
- def setBlocks(n: Int): KNN = {
- require(n > 0, "Number of blocks must be positive.")
- parameters.add(Blocks, n)
- this
- }
-
- /** Sets the Boolean variable that decides whether to use the QuadTree or not */
- def setUseQuadTree(useQuadTree: Boolean): KNN = {
- if (useQuadTree) {
- require(parameters(DistanceMetric).isInstanceOf[SquaredEuclideanDistanceMetric] ||
- parameters(DistanceMetric).isInstanceOf[EuclideanDistanceMetric])
- }
- parameters.add(UseQuadTree, useQuadTree)
- this
- }
-
- /** Parameter a user can specify if one of the training or test sets are small
- *
- * @param sizeHint cross hint tells the system which sizes to expect from the data sets
- */
- def setSizeHint(sizeHint: CrossHint): KNN = {
- parameters.add(SizeHint, sizeHint)
- this
- }
-
-}
-
-object KNN {
-
- case object K extends Parameter[Int] {
- val defaultValue: Option[Int] = Some(5)
- }
-
- case object DistanceMetric extends Parameter[DistanceMetric] {
- val defaultValue: Option[DistanceMetric] = Some(EuclideanDistanceMetric())
- }
-
- case object Blocks extends Parameter[Int] {
- val defaultValue: Option[Int] = None
- }
-
- case object UseQuadTree extends Parameter[Boolean] {
- val defaultValue: Option[Boolean] = None
- }
-
- case object SizeHint extends Parameter[CrossHint] {
- val defaultValue: Option[CrossHint] = None
- }
-
- def apply(): KNN = {
- new KNN()
- }
-
- /** [[FitOperation]] which trains a KNN based on the given training data set.
- *
- * @tparam T Subtype of [[org.apache.flink.ml.math.Vector]]
- */
- implicit def fitKNN[T <: FlinkVector : TypeInformation] = new FitOperation[KNN, T] {
- override def fit(
- instance: KNN,
- fitParameters: ParameterMap,
- input: DataSet[T]
- ): Unit = {
- val resultParameters = instance.parameters ++ fitParameters
-
- require(resultParameters.get(K).isDefined, "K is needed for calculation")
-
- val blocks = resultParameters.get(Blocks).getOrElse(input.getParallelism)
- val partitioner = FlinkMLTools.ModuloKeyPartitioner
- val inputAsVector = input.asInstanceOf[DataSet[FlinkVector]]
-
- instance.trainingSet = Some(FlinkMLTools.block(inputAsVector, blocks, Some(partitioner)))
- }
- }
-
- /** [[PredictDataSetOperation]] which calculates k-nearest neighbors of the given testing data
- * set.
- *
- * @tparam T Subtype of [[Vector]]
- * @return The given testing data set with k-nearest neighbors
- */
- implicit def predictValues[T <: FlinkVector : ClassTag : TypeInformation] = {
- new PredictDataSetOperation[KNN, T, (FlinkVector, Array[FlinkVector])] {
- override def predictDataSet(
- instance: KNN,
- predictParameters: ParameterMap,
- input: DataSet[T]
- ): DataSet[(FlinkVector, Array[FlinkVector])] = {
- val resultParameters = instance.parameters ++ predictParameters
-
- instance.trainingSet match {
- case Some(trainingSet) =>
- val k = resultParameters.get(K).get
- val blocks = resultParameters.get(Blocks).getOrElse(input.getParallelism)
- val metric = resultParameters.get(DistanceMetric).get
- val partitioner = FlinkMLTools.ModuloKeyPartitioner
-
- // attach unique id for each data
- val inputWithId: DataSet[(Long, T)] = input.zipWithUniqueId
-
- // split data into multiple blocks
- val inputSplit = FlinkMLTools.block(inputWithId, blocks, Some(partitioner))
-
- val sizeHint = resultParameters.get(SizeHint)
- val crossTuned = sizeHint match {
- case Some(hint) if hint == CrossHint.FIRST_IS_SMALL =>
- trainingSet.crossWithHuge(inputSplit)
- case Some(hint) if hint == CrossHint.SECOND_IS_SMALL =>
- trainingSet.crossWithTiny(inputSplit)
- case _ => trainingSet.cross(inputSplit)
- }
-
- // join input and training set
- val crossed = crossTuned.mapPartition {
- (iter: Iterator[(Block[FlinkVector], Block[(Long, T)])],
- out: Collector[(FlinkVector, FlinkVector, Long, Double)]) => {
- for ((training, testing) <- iter) {
- // use a quadtree if (4 ^ dim) * Ntest * log(Ntrain)
- // < Ntest * Ntrain, and distance is Euclidean
- val checkSize = math.log(4.0) * training.values.head.size +
- math.log(math.log(training.values.length)) < math.log(training.values.length)
- val checkMetric = metric match {
- case _: EuclideanDistanceMetric => true
- case _: SquaredEuclideanDistanceMetric => true
- case _ => false
- }
- val useQuadTree = resultParameters.get(UseQuadTree)
- .getOrElse(checkSize && checkMetric)
-
- if (useQuadTree) {
- knnQueryWithQuadTree(training.values, testing.values, k, metric, out)
- } else {
- knnQueryBasic(training.values, testing.values, k, metric, out)
- }
- }
- }
- }
-
- // group by input vector id and pick k nearest neighbor for each group
- val result = crossed.groupBy(2).sortGroup(3, Order.ASCENDING).reduceGroup {
- (iter: Iterator[(FlinkVector, FlinkVector, Long, Double)],
- out: Collector[(FlinkVector, Array[FlinkVector])]) => {
- if (iter.hasNext) {
- val head = iter.next()
- val key = head._2
- val neighbors: ArrayBuffer[FlinkVector] = ArrayBuffer(head._1)
-
- for ((vector, _, _, _) <- iter.take(k - 1)) {
- // we already took a first element
- neighbors += vector
- }
-
- out.collect(key, neighbors.toArray)
- }
- }
- }
-
- result
- case None => throw new RuntimeException("The KNN model has not been trained." +
- "Call first fit before calling the predict operation.")
- }
- }
- }
- }
-
- private def knnQueryWithQuadTree[T <: FlinkVector](
- training: Vector[T],
- testing: Vector[(Long, T)],
- k: Int,
- metric: DistanceMetric,
- out: Collector[(FlinkVector, FlinkVector, Long, Double)]
- ): Unit = {
- // find a bounding box
- val MinArr = Array.tabulate(training.head.size)(x => x)
- val MaxArr = Array.tabulate(training.head.size)(x => x)
-
- val minVecTrain = MinArr.map(i => training.map(x => x(i)).min - 0.01)
- val minVecTest = MinArr.map(i => testing.map(x => x._2(i)).min - 0.01)
- val maxVecTrain = MaxArr.map(i => training.map(x => x(i)).max + 0.01)
- val maxVecTest = MaxArr.map(i => testing.map(x => x._2(i)).max + 0.01)
-
- val MinVec = DenseVector(MinArr.map(i => math.min(minVecTrain(i), minVecTest(i))))
- val MaxVec = DenseVector(MinArr.map(i => math.max(maxVecTrain(i), maxVecTest(i))))
-
- // default value of max elements/box is set to max(20,k)
- val maxPerBox = math.max(k, 20)
- val trainingQuadTree = new QuadTree(MinVec, MaxVec, metric, maxPerBox)
-
- val queue = mutable.PriorityQueue[(FlinkVector, FlinkVector, Long, Double)]()(
- Ordering.by(_._4))
-
- for (v <- training) {
- trainingQuadTree.insert(v)
- }
-
- for ((id, vector) <- testing) {
- // Find siblings' objects and do local kNN there
- val siblingObjects = trainingQuadTree.searchNeighborsSiblingQueue(vector)
-
- // do KNN query on siblingObjects and get max distance of kNN then rad is good choice
- // for a neighborhood to do a refined local kNN search
- val knnSiblings = siblingObjects.map(v => metric.distance(vector, v)).sortWith(_ < _).take(k)
-
- val rad = knnSiblings.last
- val trainingFiltered = trainingQuadTree.searchNeighbors(vector, rad)
-
- for (b <- trainingFiltered) {
- // (training vector, input vector, input key, distance)
- queue.enqueue((b, vector, id, metric.distance(b, vector)))
- if (queue.size > k) {
- queue.dequeue()
- }
- }
-
- for (v <- queue) {
- out.collect(v)
- }
- }
- }
-
- private def knnQueryBasic[T <: FlinkVector](
- training: Vector[T],
- testing: Vector[(Long, T)],
- k: Int,
- metric: DistanceMetric,
- out: Collector[(FlinkVector, FlinkVector, Long, Double)]
- ): Unit = {
- val queue = mutable.PriorityQueue[(FlinkVector, FlinkVector, Long, Double)]()(
- Ordering.by(_._4))
-
- for ((id, vector) <- testing) {
- for (b <- training) {
- // (training vector, input vector, input key, distance)
- queue.enqueue((b, vector, id, metric.distance(b, vector)))
- if (queue.size > k) {
- queue.dequeue()
- }
- }
-
- for (v <- queue) {
- out.collect(v)
- }
- }
- }
-
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/nn/QuadTree.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/nn/QuadTree.scala
deleted file mode 100644
index bbccd71..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/nn/QuadTree.scala
+++ /dev/null
@@ -1,323 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.nn
-
-import org.apache.flink.ml.math.Breeze._
-import org.apache.flink.ml.math.Vector
-import org.apache.flink.ml.metrics.distances._
-
-import scala.annotation.tailrec
-import scala.collection.mutable
-
-/** n-dimensional QuadTree data structure; partitions
- * spatial data for faster queries (e.g. KNN query)
- * The skeleton of the data structure was initially
- * based off of the 2D Quadtree found here:
- * http://www.cs.trinity.edu/~mlewis/CSCI1321-F11/Code/src/util/Quadtree.scala
- *
- * Many additional methods were added to the class both for
- * efficient KNN queries and generalizing to n-dim.
- *
- * @param minVec vector of the corner of the bounding box with smallest coordinates
- * @param maxVec vector of the corner of the bounding box with smallest coordinates
- * @param distMetric metric, must be Euclidean or squareEuclidean
- * @param maxPerBox threshold for number of points in each box before slitting a box
- */
-class QuadTree(
- minVec: Vector,
- maxVec: Vector,
- distMetric: DistanceMetric,
- maxPerBox: Int) {
-
- class Node(
- center: Vector,
- width: Vector,
- var children: Seq[Node]) {
-
- val nodeElements = new mutable.ListBuffer[Vector]
-
- /** for testing purposes only; used in QuadTreeSuite.scala
- *
- * @return center and width of the box
- */
- def getCenterWidth(): (Vector, Vector) = (center, width)
-
- /** Tests whether the queryPoint is in the node, or a child of that node
- *
- * @param queryPoint a point to test
- * @return whether the given point is in the node, or a child of this node
- */
- def contains(queryPoint: Vector): Boolean = overlap(queryPoint, 0.0)
-
- /** Tests if queryPoint is within a radius of the node
- *
- * @param queryPoint a point to test
- * @param radius radius of test area
- * @return whether the given point is in the area
- */
- def overlap(queryPoint: Vector, radius: Double): Boolean = {
- (0 until queryPoint.size).forall { i =>
- (queryPoint(i) - radius < center(i) + width(i) / 2) &&
- (queryPoint(i) + radius > center(i) - width(i) / 2)
- }
- }
-
- /** Tests if queryPoint is near a node
- *
- * @param queryPoint a point to test
- * @param radius radius of covered area
- */
- def isNear(queryPoint: Vector, radius: Double): Boolean = minDist(queryPoint) < radius
-
- /** minDist is defined so that every point in the box has distance to queryPoint greater
- * than minDist (minDist adopted from "Nearest Neighbors Queries" by N. Roussopoulos et al.)
- *
- * @param queryPoint
- */
- def minDist(queryPoint: Vector): Double = {
- val minDist = (0 until queryPoint.size).map { i =>
- if (queryPoint(i) < center(i) - width(i) / 2) {
- math.pow(queryPoint(i) - center(i) + width(i) / 2, 2)
- } else if (queryPoint(i) > center(i) + width(i) / 2) {
- math.pow(queryPoint(i) - center(i) - width(i) / 2, 2)
- } else {
- 0
- }
- }.sum
-
- distMetric match {
- case _: EuclideanDistanceMetric => math.sqrt(minDist)
- case _: SquaredEuclideanDistanceMetric => minDist
- case _ => throw new IllegalArgumentException(s" Error: metric must be" +
- s" Euclidean or SquaredEuclidean!")
- }
- }
-
- /** Finds which child queryPoint lies in. node.children is a Seq[Node], and
- * [[whichChild]] finds the appropriate index of that Seq.
- *
- * @param queryPoint
- * @return
- */
- def whichChild(queryPoint: Vector): Int = {
- (0 until queryPoint.size).map { i =>
- if (queryPoint(i) > center(i)) {
- scala.math.pow(2, queryPoint.size - 1 - i).toInt
- } else {
- 0
- }
- }.sum
- }
-
- /** Makes children nodes by partitioning the box into equal sub-boxes
- * and adding a node for each sub-box
- */
- def makeChildren() {
- val centerClone = center.copy
- val cPart = partitionBox(centerClone, width)
- val mappedWidth = 0.5 * width.asBreeze
- children = cPart.map(p => new Node(p, mappedWidth.fromBreeze, null))
- }
-
- /** Recursive function that partitions a n-dim box by taking the (n-1) dimensional
- * plane through the center of the box keeping the n-th coordinate fixed,
- * then shifting it in the n-th direction up and down
- * and recursively applying partitionBox to the two shifted (n-1) dimensional planes.
- *
- * @param center the center of the box
- * @param width a vector of lengths of each dimension of the box
- * @return
- */
- def partitionBox(center: Vector, width: Vector): Seq[Vector] = {
- @tailrec
- def partitionHelper(box: Seq[Vector], dim: Int): Seq[Vector] = {
- if (dim >= width.size) {
- box
- } else {
- val newBox = box.flatMap { vector =>
- val (up, down) = (vector.copy, vector)
- up.update(dim, up(dim) - width(dim) / 4)
- down.update(dim, down(dim) + width(dim) / 4)
-
- Seq(up, down)
- }
- partitionHelper(newBox, dim + 1)
- }
- }
- partitionHelper(Seq(center), 0)
- }
- }
-
-
- val root = new Node(((minVec.asBreeze + maxVec.asBreeze) * 0.5).fromBreeze,
- (maxVec.asBreeze - minVec.asBreeze).fromBreeze, null)
-
- /** Prints tree for testing/debugging */
- def printTree(): Unit = {
- def printTreeRecur(node: Node) {
- if (node.children != null) {
- for (c <- node.children) {
- printTreeRecur(c)
- }
- } else {
- println("printing tree: n.nodeElements " + node.nodeElements)
- }
- }
-
- printTreeRecur(root)
- }
-
- /** Recursively adds an object to the tree
- *
- * @param queryPoint an object which is added
- */
- def insert(queryPoint: Vector) = {
- def insertRecur(queryPoint: Vector, node: Node): Unit = {
- if (node.children == null) {
- if (node.nodeElements.length < maxPerBox) {
- node.nodeElements += queryPoint
- } else {
- node.makeChildren()
- for (o <- node.nodeElements) {
- insertRecur(o, node.children(node.whichChild(o)))
- }
- node.nodeElements.clear()
- insertRecur(queryPoint, node.children(node.whichChild(queryPoint)))
- }
- } else {
- insertRecur(queryPoint, node.children(node.whichChild(queryPoint)))
- }
- }
-
- insertRecur(queryPoint, root)
- }
-
- /** Used to zoom in on a region near a test point for a fast KNN query.
- * This capability is used in the KNN query to find k "near" neighbors n_1,...,n_k, from
- * which one computes the max distance D_s to queryPoint. D_s is then used during the
- * kNN query to find all points within a radius D_s of queryPoint using searchNeighbors.
- * To find the "near" neighbors, a min-heap is defined on the leaf nodes of the leaf
- * nodes of the minimal bounding box of the queryPoint. The priority of a leaf node
- * is an appropriate notion of the distance between the test point and the node,
- * which is defined by minDist(queryPoint),
- *
- * @param queryPoint a test point for which the method finds the minimal bounding
- * box that queryPoint lies in and returns elements in that boxes
- * siblings' leaf nodes
- */
- def searchNeighborsSiblingQueue(queryPoint: Vector): mutable.ListBuffer[Vector] = {
- val ret = new mutable.ListBuffer[Vector]
- // edge case when the main box has not been partitioned at all
- if (root.children == null) {
- root.nodeElements.clone()
- } else {
- val nodeQueue = new mutable.PriorityQueue[(Double, Node)]()(Ordering.by(x => x._1))
- searchRecurSiblingQueue(queryPoint, root, nodeQueue)
-
- var count = 0
- while (count < maxPerBox) {
- val dq = nodeQueue.dequeue()
- if (dq._2.nodeElements.nonEmpty) {
- ret ++= dq._2.nodeElements
- count += dq._2.nodeElements.length
- }
- }
- ret
- }
- }
-
- /**
- *
- * @param queryPoint point under consideration
- * @param node node that queryPoint lies in
- * @param nodeQueue defined in searchSiblingQueue, this stores nodes based on their
- * distance to node as defined by minDist
- */
- private def searchRecurSiblingQueue(
- queryPoint: Vector,
- node: Node,
- nodeQueue: mutable.PriorityQueue[(Double, Node)]
- ): Unit = {
- if (node.children != null) {
- for (child <- node.children; if child.contains(queryPoint)) {
- if (child.children == null) {
- for (c <- node.children) {
- minNodes(queryPoint, c, nodeQueue)
- }
- } else {
- searchRecurSiblingQueue(queryPoint, child, nodeQueue)
- }
- }
- }
- }
-
- /** Goes down to minimal bounding box of queryPoint, and add elements to nodeQueue
- *
- * @param queryPoint point under consideration
- * @param node node that queryPoint lies in
- * @param nodeQueue [[mutable.PriorityQueue]] that stores all points in minimal bounding box
- * of queryPoint
- */
- private def minNodes(
- queryPoint: Vector,
- node: Node,
- nodeQueue: mutable.PriorityQueue[(Double, Node)]
- ): Unit = {
- if (node.children == null) {
- nodeQueue += ((-node.minDist(queryPoint), node))
- } else {
- for (c <- node.children) {
- minNodes(queryPoint, c, nodeQueue)
- }
- }
- }
-
- /** Finds all objects within a neighborhood of queryPoint of a specified radius
- * scope is modified from original 2D version in:
- * http://www.cs.trinity.edu/~mlewis/CSCI1321-F11/Code/src/util/Quadtree.scala
- *
- * original version only looks in minimal box; for the KNN Query, we look at
- * all nearby boxes. The radius is determined from searchNeighborsSiblingQueue
- * by defining a min-heap on the leaf nodes
- *
- * @param queryPoint a point which is center
- * @param radius radius of scope
- * @return all points within queryPoint with given radius
- */
- def searchNeighbors(queryPoint: Vector, radius: Double): mutable.ListBuffer[Vector] = {
- def searchRecur(
- queryPoint: Vector,
- radius: Double,
- node: Node,
- ret: mutable.ListBuffer[Vector]
- ): Unit = {
- if (node.children == null) {
- ret ++= node.nodeElements
- } else {
- for (child <- node.children; if child.isNear(queryPoint, radius)) {
- searchRecur(queryPoint, radius, child, ret)
- }
- }
- }
-
- val ret = new mutable.ListBuffer[Vector]
- searchRecur(queryPoint, radius, root, ret)
- ret
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala
deleted file mode 100644
index fbb3a31..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/GradientDescent.scala
+++ /dev/null
@@ -1,297 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.flink.ml.optimization
-
-import org.apache.flink.api.scala._
-import org.apache.flink.ml.common._
-import org.apache.flink.ml.math._
-import org.apache.flink.ml.optimization.IterativeSolver._
-import org.apache.flink.ml.optimization.LearningRateMethod.LearningRateMethodTrait
-import org.apache.flink.ml.optimization.Solver._
-import org.apache.flink.ml._
-
-/** Base class which performs Stochastic Gradient Descent optimization using mini batches.
- *
- * For each labeled vector in a mini batch the gradient is computed and added to a partial
- * gradient. The partial gradients are then summed and divided by the size of the batches. The
- * average gradient is then used to updated the weight values, including regularization.
- *
- * At the moment, the whole partition is used for SGD, making it effectively a batch gradient
- * descent. Once a sampling operator has been introduced, the algorithm can be optimized
- *
- * The parameters to tune the algorithm are:
- * [[Solver.LossFunction]] for the loss function to be used,
- * [[Solver.RegularizationPenaltyValue]] for the regularization penalty.
- * [[Solver.RegularizationConstant]] for the regularization parameter,
- * [[IterativeSolver.Iterations]] for the maximum number of iteration,
- * [[IterativeSolver.LearningRate]] for the learning rate used.
- * [[IterativeSolver.ConvergenceThreshold]] when provided the algorithm will
- * stop the iterations if the relative change in the value of the objective
- * function between successive iterations is is smaller than this value.
- * [[IterativeSolver.LearningRateMethodValue]] determines functional form of
- * effective learning rate.
- */
-class GradientDescent extends IterativeSolver {
-
- /** Provides a solution for the given optimization problem
- *
- * @param data A Dataset of LabeledVector (label, features) pairs
- * @param initialWeights The initial weights that will be optimized
- * @return The weights, optimized for the provided data.
- */
- override def optimize(
- data: DataSet[LabeledVector],
- initialWeights: Option[DataSet[WeightVector]]): DataSet[WeightVector] = {
-
- val numberOfIterations: Int = parameters(Iterations)
- val convergenceThresholdOption: Option[Double] = parameters.get(ConvergenceThreshold)
- val lossFunction = parameters(LossFunction)
- val learningRate = parameters(LearningRate)
- val regularizationPenalty = parameters(RegularizationPenaltyValue)
- val regularizationConstant = parameters(RegularizationConstant)
- val learningRateMethod = parameters(LearningRateMethodValue)
-
- // Initialize weights
- val initialWeightsDS: DataSet[WeightVector] = createInitialWeightsDS(initialWeights, data)
-
- // Perform the iterations
- convergenceThresholdOption match {
- // No convergence criterion
- case None =>
- optimizeWithoutConvergenceCriterion(
- data,
- initialWeightsDS,
- numberOfIterations,
- regularizationPenalty,
- regularizationConstant,
- learningRate,
- lossFunction,
- learningRateMethod)
- case Some(convergence) =>
- optimizeWithConvergenceCriterion(
- data,
- initialWeightsDS,
- numberOfIterations,
- regularizationPenalty,
- regularizationConstant,
- learningRate,
- convergence,
- lossFunction,
- learningRateMethod)
- }
- }
-
- def optimizeWithConvergenceCriterion(
- dataPoints: DataSet[LabeledVector],
- initialWeightsDS: DataSet[WeightVector],
- numberOfIterations: Int,
- regularizationPenalty: RegularizationPenalty,
- regularizationConstant: Double,
- learningRate: Double,
- convergenceThreshold: Double,
- lossFunction: LossFunction,
- learningRateMethod: LearningRateMethodTrait)
- : DataSet[WeightVector] = {
- // We have to calculate for each weight vector the sum of squared residuals,
- // and then sum them and apply regularization
- val initialLossSumDS = calculateLoss(dataPoints, initialWeightsDS, lossFunction)
-
- // Combine weight vector with the current loss
- val initialWeightsWithLossSum = initialWeightsDS.mapWithBcVariable(initialLossSumDS){
- (weights, loss) => (weights, loss)
- }
-
- val resultWithLoss = initialWeightsWithLossSum.iterateWithTermination(numberOfIterations) {
- weightsWithPreviousLossSum =>
-
- // Extract weight vector and loss
- val previousWeightsDS = weightsWithPreviousLossSum.map{_._1}
- val previousLossSumDS = weightsWithPreviousLossSum.map{_._2}
-
- val currentWeightsDS = SGDStep(
- dataPoints,
- previousWeightsDS,
- lossFunction,
- regularizationPenalty,
- regularizationConstant,
- learningRate,
- learningRateMethod)
-
- val currentLossSumDS = calculateLoss(dataPoints, currentWeightsDS, lossFunction)
-
- // Check if the relative change in the loss is smaller than the
- // convergence threshold. If yes, then terminate i.e. return empty termination data set
- val termination = previousLossSumDS.filterWithBcVariable(currentLossSumDS){
- (previousLoss, currentLoss) => {
- if (previousLoss <= 0) {
- false
- } else {
- scala.math.abs((previousLoss - currentLoss)/previousLoss) >= convergenceThreshold
- }
- }
- }
-
- // Result for new iteration
- (currentWeightsDS.mapWithBcVariable(currentLossSumDS)((w, l) => (w, l)), termination)
- }
- // Return just the weights
- resultWithLoss.map{_._1}
- }
-
- def optimizeWithoutConvergenceCriterion(
- data: DataSet[LabeledVector],
- initialWeightsDS: DataSet[WeightVector],
- numberOfIterations: Int,
- regularizationPenalty: RegularizationPenalty,
- regularizationConstant: Double,
- learningRate: Double,
- lossFunction: LossFunction,
- optimizationMethod: LearningRateMethodTrait)
- : DataSet[WeightVector] = {
- initialWeightsDS.iterate(numberOfIterations) {
- weightVectorDS => {
- SGDStep(data,
- weightVectorDS,
- lossFunction,
- regularizationPenalty,
- regularizationConstant,
- learningRate,
- optimizationMethod)
- }
- }
- }
-
- /** Performs one iteration of Stochastic Gradient Descent using mini batches
- *
- * @param data A Dataset of LabeledVector (label, features) pairs
- * @param currentWeights A Dataset with the current weights to be optimized as its only element
- * @param lossFunction The loss function to be used
- * @param regularizationPenalty The regularization penalty to be used
- * @param regularizationConstant The regularization parameter
- * @param learningRate The effective step size for this iteration
- * @param learningRateMethod The learning rate used
- *
- * @return A Dataset containing the weights after one stochastic gradient descent step
- */
- private def SGDStep(
- data: DataSet[(LabeledVector)],
- currentWeights: DataSet[WeightVector],
- lossFunction: LossFunction,
- regularizationPenalty: RegularizationPenalty,
- regularizationConstant: Double,
- learningRate: Double,
- learningRateMethod: LearningRateMethodTrait)
- : DataSet[WeightVector] = {
-
- data.mapWithBcVariable(currentWeights){
- (data, weightVector) => (lossFunction.gradient(data, weightVector), 1)
- }.reduce{
- (left, right) =>
- val (leftGradVector, leftCount) = left
- val (rightGradVector, rightCount) = right
-
- // make the left gradient dense so that the following reduce operations (left fold) reuse
- // it. This strongly depends on the underlying implementation of the ReduceDriver which
- // always passes the new input element as the second parameter
- val result = leftGradVector.weights match {
- case d: DenseVector => d
- case s: SparseVector => s.toDenseVector
- }
-
- // Add the right gradient to the result
- BLAS.axpy(1.0, rightGradVector.weights, result)
- val gradients = WeightVector(
- result, leftGradVector.intercept + rightGradVector.intercept)
-
- (gradients , leftCount + rightCount)
- }.mapWithBcVariableIteration(currentWeights){
- (gradientCount, weightVector, iteration) => {
- val (WeightVector(weights, intercept), count) = gradientCount
-
- BLAS.scal(1.0/count, weights)
-
- val gradient = WeightVector(weights, intercept/count)
- val effectiveLearningRate = learningRateMethod.calculateLearningRate(
- learningRate,
- iteration,
- regularizationConstant)
-
- val newWeights = takeStep(
- weightVector.weights,
- gradient.weights,
- regularizationPenalty,
- regularizationConstant,
- effectiveLearningRate)
-
- WeightVector(
- newWeights,
- weightVector.intercept - effectiveLearningRate * gradient.intercept)
- }
- }
- }
-
- /** Calculates the new weights based on the gradient
- *
- * @param weightVector The weights to be updated
- * @param gradient The gradient according to which we will update the weights
- * @param regularizationPenalty The regularization penalty to apply
- * @param regularizationConstant The regularization parameter
- * @param learningRate The effective step size for this iteration
- * @return Updated weights
- */
- def takeStep(
- weightVector: Vector,
- gradient: Vector,
- regularizationPenalty: RegularizationPenalty,
- regularizationConstant: Double,
- learningRate: Double
- ): Vector = {
- regularizationPenalty.takeStep(weightVector, gradient, regularizationConstant, learningRate)
- }
-
- /** Calculates the regularized loss, from the data and given weights.
- *
- * @param data A Dataset of LabeledVector (label, features) pairs
- * @param weightDS A Dataset with the current weights to be optimized as its only element
- * @param lossFunction The loss function to be used
- * @return A Dataset with the regularized loss as its only element
- */
- private def calculateLoss(
- data: DataSet[LabeledVector],
- weightDS: DataSet[WeightVector],
- lossFunction: LossFunction)
- : DataSet[Double] = {
- data.mapWithBcVariable(weightDS){
- (data, weightVector) => (lossFunction.loss(data, weightVector), 1)
- }.reduce{
- (left, right) => (left._1 + right._1, left._2 + right._2)
- }.map {
- lossCount => lossCount._1 / lossCount._2
- }
- }
-}
-
-
-/** Implementation of a Gradient Descent solver.
- *
- */
-object GradientDescent {
- def apply() = new GradientDescent
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala
deleted file mode 100644
index bf96cac..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/LossFunction.scala
+++ /dev/null
@@ -1,96 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.optimization
-
-import org.apache.flink.ml.common.{WeightVector, LabeledVector}
-import org.apache.flink.ml.math.BLAS
-
-/** Abstract class that implements some of the functionality for common loss functions
- *
- * A loss function determines the loss term `L(w)` of the objective function `f(w) = L(w) +
- * lambda*R(w)` for prediction tasks, the other being regularization, `R(w)`.
- *
- * The regularization is specific to the used optimization algorithm and, thus, implemented there.
- *
- * We currently only support differentiable loss functions, in the future this class
- * could be changed to DiffLossFunction in order to support other types, such as absolute loss.
- */
-trait LossFunction extends Serializable {
-
- /** Calculates the loss given the prediction and label value
- *
- * @param dataPoint
- * @param weightVector
- * @return
- */
- def loss(dataPoint: LabeledVector, weightVector: WeightVector): Double = {
- lossGradient(dataPoint, weightVector)._1
- }
-
- /** Calculates the gradient of the loss function given a data point and weight vector
- *
- * @param dataPoint
- * @param weightVector
- * @return
- */
- def gradient(dataPoint: LabeledVector, weightVector: WeightVector): WeightVector = {
- lossGradient(dataPoint, weightVector)._2
- }
-
- /** Calculates the gradient as well as the loss given a data point and the weight vector
- *
- * @param dataPoint
- * @param weightVector
- * @return
- */
- def lossGradient(dataPoint: LabeledVector, weightVector: WeightVector): (Double, WeightVector)
-}
-
-/** Generic loss function which lets you build a loss function out of the [[PartialLossFunction]]
- * and the [[PredictionFunction]].
- *
- * @param partialLossFunction
- * @param predictionFunction
- */
-case class GenericLossFunction(
- partialLossFunction: PartialLossFunction,
- predictionFunction: PredictionFunction)
- extends LossFunction {
-
- /** Calculates the gradient as well as the loss given a data point and the weight vector
- *
- * @param dataPoint
- * @param weightVector
- * @return
- */
- def lossGradient(dataPoint: LabeledVector, weightVector: WeightVector): (Double, WeightVector) = {
- val prediction = predictionFunction.predict(dataPoint.vector, weightVector)
-
- val loss = partialLossFunction.loss(prediction, dataPoint.label)
-
- val lossDerivative = partialLossFunction.derivative(prediction, dataPoint.label)
-
- val WeightVector(weightGradient, interceptGradient) =
- predictionFunction.gradient(dataPoint.vector, weightVector)
-
- BLAS.scal(lossDerivative, weightGradient)
-
- (loss, WeightVector(weightGradient, lossDerivative * interceptGradient))
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PartialLossFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PartialLossFunction.scala
deleted file mode 100644
index 10f7e00..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PartialLossFunction.scala
+++ /dev/null
@@ -1,154 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.optimization
-
-/** Represents loss functions which can be used with the [[GenericLossFunction]].
- *
- */
-trait PartialLossFunction extends Serializable {
- /** Calculates the loss depending on the label and the prediction
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The loss
- */
- def loss(prediction: Double, label: Double): Double
-
- /** Calculates the derivative of the [[PartialLossFunction]]
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The derivative of the loss function
- */
- def derivative(prediction: Double, label: Double): Double
-}
-
-/** Squared loss function which can be used with the [[GenericLossFunction]]
- *
- * The [[SquaredLoss]] function implements `1/2 (prediction - label)^2`
- */
-object SquaredLoss extends PartialLossFunction {
-
- /** Calculates the loss depending on the label and the prediction
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The loss
- */
- override def loss(prediction: Double, label: Double): Double = {
- 0.5 * (prediction - label) * (prediction - label)
- }
-
- /** Calculates the derivative of the [[PartialLossFunction]]
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The derivative of the loss function
- */
- override def derivative(prediction: Double, label: Double): Double = {
- prediction - label
- }
-}
-
-/** Logistic loss function which can be used with the [[GenericLossFunction]]
- *
- *
- * The [[LogisticLoss]] function implements `log(1 + -exp(prediction*label))`
- * for binary classification with label in {-1, 1}
- */
-object LogisticLoss extends PartialLossFunction {
-
- /** Calculates the loss depending on the label and the prediction
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The loss
- */
- override def loss(prediction: Double, label: Double): Double = {
- val z = prediction * label
-
- // based on implementation in scikit-learn
- // approximately equal and saves the computation of the log
- if (z > 18) {
- math.exp(-z)
- }
- else if (z < -18) {
- -z
- }
- else {
- math.log(1 + math.exp(-z))
- }
- }
-
- /** Calculates the derivative of the loss function with respect to the prediction
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The derivative of the loss function
- */
- override def derivative(prediction: Double, label: Double): Double = {
- val z = prediction * label
-
- // based on implementation in scikit-learn
- // approximately equal and saves the computation of the log
- if (z > 18) {
- label * math.exp(-z)
- }
- else if (z < -18) {
- -label
- }
- else {
- -label/(math.exp(z) + 1)
- }
- }
-}
-
-/** Hinge loss function which can be used with the [[GenericLossFunction]]
- *
- * The [[HingeLoss]] function implements `max(0, 1 - prediction*label)`
- * for binary classification with label in {-1, 1}
- */
-object HingeLoss extends PartialLossFunction {
- /** Calculates the loss for a given prediction/truth pair
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The loss
- */
- override def loss(prediction: Double, label: Double): Double = {
- val z = prediction * label
- math.max(0, 1 - z)
- }
-
- /** Calculates the derivative of the loss function with respect to the prediction
- *
- * @param prediction The predicted value
- * @param label The true value
- * @return The derivative of the loss function
- */
- override def derivative(prediction: Double, label: Double): Double = {
- val z = prediction * label
- if (z <= 1) {
- -label
- }
- else {
- 0
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala
deleted file mode 100644
index 38f340a..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/PredictionFunction.scala
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.optimization
-
-import org.apache.flink.ml.common.WeightVector
-import org.apache.flink.ml.math.{Vector => FlinkVector, BLAS}
-
-/** An abstract class for prediction functions to be used in optimization **/
-abstract class PredictionFunction extends Serializable {
- def predict(features: FlinkVector, weights: WeightVector): Double
-
- def gradient(features: FlinkVector, weights: WeightVector): WeightVector
-}
-
-/** A linear prediction function **/
-object LinearPrediction extends PredictionFunction {
- override def predict(features: FlinkVector, weightVector: WeightVector): Double = {
- BLAS.dot(features, weightVector.weights) + weightVector.intercept
- }
-
- override def gradient(features: FlinkVector, weights: WeightVector): WeightVector = {
- WeightVector(features.copy, 1)
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/RegularizationPenalty.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/RegularizationPenalty.scala
deleted file mode 100644
index 1ed59ab..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/RegularizationPenalty.scala
+++ /dev/null
@@ -1,219 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.optimization
-
-import org.apache.flink.ml.math.{Vector, BLAS}
-import org.apache.flink.ml.math.Breeze._
-import breeze.linalg.{norm => BreezeNorm}
-
-/** Represents a type of regularization penalty
- *
- * Regularization penalties are used to restrict the optimization problem to solutions with
- * certain desirable characteristics, such as sparsity for the L1 penalty, or penalizing large
- * weights for the L2 penalty.
- *
- * The regularization term, `R(w)` is added to the objective function, `f(w) = L(w) + lambda*R(w)`
- * where lambda is the regularization parameter used to tune the amount of regularization applied.
- */
-trait RegularizationPenalty extends Serializable {
-
- /** Calculates the new weights based on the gradient and regularization penalty
- *
- * Weights are updated using the gradient descent step `w - learningRate * gradient`
- * with `w` being the weight vector.
- *
- * @param weightVector The weights to be updated
- * @param gradient The gradient used to update the weights
- * @param regularizationConstant The regularization parameter to be applied
- * @param learningRate The effective step size for this iteration
- * @return Updated weights
- */
- def takeStep(
- weightVector: Vector,
- gradient: Vector,
- regularizationConstant: Double,
- learningRate: Double)
- : Vector
-
- /** Adds regularization to the loss value
- *
- * @param oldLoss The loss to be updated
- * @param weightVector The weights used to update the loss
- * @param regularizationConstant The regularization parameter to be applied
- * @return Updated loss
- */
- def regLoss(oldLoss: Double, weightVector: Vector, regularizationConstant: Double): Double
-
-}
-
-
-/** `L_2` regularization penalty.
- *
- * The regularization function is the square of the L2 norm `1/2*||w||_2^2`
- * with `w` being the weight vector. The function penalizes large weights,
- * favoring solutions with more small weights rather than few large ones.
- */
-object L2Regularization extends RegularizationPenalty {
-
- /** Calculates the new weights based on the gradient and L2 regularization penalty
- *
- * The updated weight is `w - learningRate * (gradient + lambda * w)` where
- * `w` is the weight vector, and `lambda` is the regularization parameter.
- *
- * @param weightVector The weights to be updated
- * @param gradient The gradient according to which we will update the weights
- * @param regularizationConstant The regularization parameter to be applied
- * @param learningRate The effective step size for this iteration
- * @return Updated weights
- */
- override def takeStep(
- weightVector: Vector,
- gradient: Vector,
- regularizationConstant: Double,
- learningRate: Double)
- : Vector = {
- // add the gradient of the L2 regularization
- BLAS.axpy(regularizationConstant, weightVector, gradient)
-
- // update the weights according to the learning rate
- BLAS.axpy(-learningRate, gradient, weightVector)
-
- weightVector
- }
-
- /** Adds regularization to the loss value
- *
- * The updated loss is `oldLoss + lambda * 1/2*||w||_2^2` where
- * `w` is the weight vector, and `lambda` is the regularization parameter
- *
- * @param oldLoss The loss to be updated
- * @param weightVector The weights used to update the loss
- * @param regularizationConstant The regularization parameter to be applied
- * @return Updated loss
- */
- override def regLoss(oldLoss: Double, weightVector: Vector, regularizationConstant: Double)
- : Double = {
- val squareNorm = BLAS.dot(weightVector, weightVector)
- oldLoss + regularizationConstant * 0.5 * squareNorm
- }
-}
-
-/** `L_1` regularization penalty.
- *
- * The regularization function is the `L1` norm `||w||_1` with `w` being the weight vector.
- * The `L_1` penalty can be used to drive a number of the solution coefficients to 0, thereby
- * producing sparse solutions.
- *
- */
-object L1Regularization extends RegularizationPenalty {
-
- /** Calculates the new weights based on the gradient and L1 regularization penalty
- *
- * Uses the proximal gradient method with L1 regularization to update weights.
- * The updated weight `w - learningRate * gradient` is shrunk towards zero
- * by applying the proximal operator `signum(w) * max(0.0, abs(w) - shrinkageVal)`
- * where `w` is the weight vector, `lambda` is the regularization parameter,
- * and `shrinkageVal` is `lambda*learningRate`.
- *
- * @param weightVector The weights to be updated
- * @param gradient The gradient according to which we will update the weights
- * @param regularizationConstant The regularization parameter to be applied
- * @param learningRate The effective step size for this iteration
- * @return Updated weights
- */
- override def takeStep(
- weightVector: Vector,
- gradient: Vector,
- regularizationConstant: Double,
- learningRate: Double)
- : Vector = {
- // Update weight vector with gradient.
- BLAS.axpy(-learningRate, gradient, weightVector)
-
- // Apply proximal operator (soft thresholding)
- val shrinkageVal = regularizationConstant * learningRate
- var i = 0
- while (i < weightVector.size) {
- val wi = weightVector(i)
- weightVector(i) = math.signum(wi) *
- math.max(0.0, math.abs(wi) - shrinkageVal)
- i += 1
- }
-
- weightVector
- }
-
- /** Adds regularization to the loss value
- *
- * The updated loss is `oldLoss + lambda * ||w||_1` where
- * `w` is the weight vector and `lambda` is the regularization parameter
- *
- * @param oldLoss The loss to be updated
- * @param weightVector The weights used to update the loss
- * @param regularizationConstant The regularization parameter to be applied
- * @return Updated loss
- */
- override def regLoss(oldLoss: Double, weightVector: Vector, regularizationConstant: Double)
- : Double = {
- val norm = BreezeNorm(weightVector.asBreeze, 1.0)
- oldLoss + norm * regularizationConstant
- }
-}
-
-/** No regularization penalty.
- *
- */
-object NoRegularization extends RegularizationPenalty {
-
- /** Calculates the new weights based on the gradient
- *
- * The updated weight is `w - learningRate *gradient` where `w` is the weight vector
- *
- * @param weightVector The weights to be updated
- * @param gradient The gradient according to which we will update the weights
- * @param regularizationConstant The regularization parameter which is ignored
- * @param learningRate The effective step size for this iteration
- * @return Updated weights
- */
- override def takeStep(
- weightVector: Vector,
- gradient: Vector,
- regularizationConstant: Double,
- learningRate: Double)
- : Vector = {
- // Update the weight vector
- BLAS.axpy(-learningRate, gradient, weightVector)
- weightVector
- }
-
- /**
- * Returns the unmodified loss value
- *
- * The updated loss is `oldLoss`
- *
- * @param oldLoss The loss to be updated
- * @param weightVector The weights used to update the loss
- * @param regularizationParameter The regularization parameter which is ignored
- * @return Updated loss
- */
- override def regLoss(oldLoss: Double, weightVector: Vector, regularizationParameter: Double)
- : Double = {
- oldLoss
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala
deleted file mode 100644
index 761620d..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/optimization/Solver.scala
+++ /dev/null
@@ -1,233 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.optimization
-
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.ml.common._
-import org.apache.flink.ml.math.{SparseVector, DenseVector}
-import org.apache.flink.api.scala._
-import org.apache.flink.ml.optimization.IterativeSolver._
-import org.apache.flink.ml.optimization.LearningRateMethod.LearningRateMethodTrait
-
-/** Base class for optimization algorithms
- *
- */
-abstract class Solver extends Serializable with WithParameters {
- import Solver._
-
- /** Provides a solution for the given optimization problem
- *
- * @param data A Dataset of LabeledVector (input, output) pairs
- * @param initialWeights The initial weight that will be optimized
- * @return A Vector of weights optimized to the given problem
- */
- def optimize(
- data: DataSet[LabeledVector],
- initialWeights: Option[DataSet[WeightVector]])
- : DataSet[WeightVector]
-
- /** Creates initial weights vector, creating a DataSet with a WeightVector element
- *
- * @param initialWeights An Option that may contain an initial set of weights
- * @param data The data for which we optimize the weights
- * @return A DataSet containing a single WeightVector element
- */
- def createInitialWeightsDS(initialWeights: Option[DataSet[WeightVector]],
- data: DataSet[LabeledVector]): DataSet[WeightVector] = {
- // TODO: Faster way to do this?
- val dimensionsDS = data.map(_.vector.size).reduce((a, b) => b)
-
- initialWeights match {
- // Ensure provided weight vector is a DenseVector
- case Some(wvDS) =>
- wvDS.map {
- wv => {
- val denseWeights = wv.weights match {
- case dv: DenseVector => dv
- case sv: SparseVector => sv.toDenseVector
- }
- WeightVector(denseWeights, wv.intercept)
- }
- }
- case None => createInitialWeightVector(dimensionsDS)
- }
- }
-
- /** Creates a DataSet with one zero vector. The zero vector has dimension d, which is given
- * by the dimensionDS.
- *
- * @param dimensionDS DataSet with one element d, denoting the dimension of the returned zero
- * vector
- * @return DataSet of a zero vector of dimension d
- */
- def createInitialWeightVector(dimensionDS: DataSet[Int]): DataSet[WeightVector] = {
- dimensionDS.map {
- dimension =>
- val values = Array.fill(dimension)(0.0)
- WeightVector(DenseVector(values), 0.0)
- }
- }
-
- //Setters for parameters
- // TODO(tvas): Provide an option to fit an intercept or not
- def setLossFunction(lossFunction: LossFunction): this.type = {
- parameters.add(LossFunction, lossFunction)
- this
- }
-
- def setRegularizationConstant(regularizationConstant: Double): this.type = {
- parameters.add(RegularizationConstant, regularizationConstant)
- this
- }
-
- def setRegularizationPenalty(regularizationPenalty: RegularizationPenalty) : this.type = {
- parameters.add(RegularizationPenaltyValue, regularizationPenalty)
- this
- }
-}
-
-object Solver {
- // Define parameters for Solver
- case object LossFunction extends Parameter[LossFunction] {
- // TODO(tvas): Should depend on problem, here is where differentiating between classification
- // and regression could become useful
- val defaultValue = None
- }
-
- case object RegularizationConstant extends Parameter[Double] {
- val defaultValue = Some(0.0001) // TODO(tvas): Properly initialize this, ensure Parameter > 0!
- }
-
- case object RegularizationPenaltyValue extends Parameter[RegularizationPenalty] {
- val defaultValue = Some(NoRegularization)
- }
-}
-
-/** An abstract class for iterative optimization algorithms
- *
- * See [[https://en.wikipedia.org/wiki/Iterative_method Iterative Methods on Wikipedia]] for more
- * info
- */
-abstract class IterativeSolver() extends Solver {
-
- //Setters for parameters
- def setIterations(iterations: Int): this.type = {
- parameters.add(Iterations, iterations)
- this
- }
-
- def setStepsize(stepsize: Double): this.type = {
- parameters.add(LearningRate, stepsize)
- this
- }
-
- def setConvergenceThreshold(convergenceThreshold: Double): this.type = {
- parameters.add(ConvergenceThreshold, convergenceThreshold)
- this
- }
-
- def setLearningRateMethod(learningRateMethod: LearningRateMethodTrait): this.type = {
- parameters.add(LearningRateMethodValue, learningRateMethod)
- this
- }
-}
-
-object IterativeSolver {
-
- val MAX_DLOSS: Double = 1e12
-
- // Define parameters for IterativeSolver
- case object LearningRate extends Parameter[Double] {
- val defaultValue = Some(0.1)
- }
-
- case object Iterations extends Parameter[Int] {
- val defaultValue = Some(10)
- }
-
- case object ConvergenceThreshold extends Parameter[Double] {
- val defaultValue = None
- }
-
- case object LearningRateMethodValue extends Parameter[LearningRateMethodTrait] {
- val defaultValue = Some(LearningRateMethod.Default)
- }
-}
-
-object LearningRateMethod {
-
- sealed trait LearningRateMethodTrait extends Serializable {
- def calculateLearningRate(
- initialLearningRate: Double,
- iteration: Int,
- regularizationConstant: Double)
- : Double
- }
-
- object Default extends LearningRateMethodTrait {
- override def calculateLearningRate(
- initialLearningRate: Double,
- iteration: Int,
- regularizationConstant: Double)
- : Double = {
- initialLearningRate / Math.sqrt(iteration)
- }
- }
-
- object Constant extends LearningRateMethodTrait {
- override def calculateLearningRate(
- initialLearningRate: Double,
- iteration: Int,
- regularizationConstant: Double)
- : Double = {
- initialLearningRate
- }
- }
-
- case class Bottou(optimalInit: Double) extends LearningRateMethodTrait {
- override def calculateLearningRate(
- initialLearningRate: Double,
- iteration: Int,
- regularizationConstant: Double)
- : Double = {
- 1 / (regularizationConstant * (optimalInit + iteration - 1))
- }
- }
-
- case class InvScaling(decay: Double) extends LearningRateMethodTrait {
- override def calculateLearningRate(
- initialLearningRate: Double,
- iteration: Int,
- regularizationConstant: Double)
- : Double = {
- initialLearningRate / Math.pow(iteration, decay)
- }
- }
-
- case class Xu(decay: Double) extends LearningRateMethodTrait {
- override def calculateLearningRate(
- initialLearningRate: Double,
- iteration: Int,
- regularizationConstant: Double)
- : Double = {
- initialLearningRate *
- Math.pow(1 + regularizationConstant * initialLearningRate * iteration, -decay)
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/outlier/StochasticOutlierSelection.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/outlier/StochasticOutlierSelection.scala
deleted file mode 100644
index 2ff46db..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/outlier/StochasticOutlierSelection.scala
+++ /dev/null
@@ -1,383 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.outlier
-
-/** An implementation of the Stochastic Outlier Selection algorithm by Jeroen Jansen
- *
- * For more information about SOS, see https://github.com/jeroenjanssens/sos
- * J.H.M. Janssens, F. Huszar, E.O. Postma, and H.J. van den Herik. Stochastic
- * Outlier Selection. Technical Report TiCC TR 2012-001, Tilburg University,
- * Tilburg, the Netherlands, 2012.
- *
- * @example
- * {{{
- * val data = env.fromCollection(List(
- * LabeledVector(0.0, DenseVector(1.0, 1.0)),
- * LabeledVector(1.0, DenseVector(2.0, 1.0)),
- * LabeledVector(2.0, DenseVector(1.0, 2.0)),
- * LabeledVector(3.0, DenseVector(2.0, 2.0)),
- * LabeledVector(4.0, DenseVector(5.0, 8.0)) // The outlier!
- * ))
- *
- * val sos = new StochasticOutlierSelection().setPerplexity(3)
- *
- * val outputVector = sos
- * .transform(data)
- * .collect()
- *
- * val expectedOutputVector = Map(
- * 0 -> 0.2790094479202896,
- * 1 -> 0.25775014551682535,
- * 2 -> 0.22136130977995766,
- * 3 -> 0.12707053787018444,
- * 4 -> 0.9922779902453757 // The outlier!
- * )
- *
- * outputVector.foreach(output => expectedOutputVector(output._1) should be(output._2))
- * }}}
- *
- * =Parameters=
- *
- * - [[org.apache.flink.ml.outlier.StochasticOutlierSelection.Perplexity]]:
- * Perplexity can be interpreted as the k in k-nearest neighbor algorithms. The difference is
- * in SOS being a neighbor is not a binary property, but a probabilistic one, and therefore it
- * a real number. Must be between 1 and n-1, where n is the number of points.
- * (Default value: '''30''')
- *
- * - [[org.apache.flink.ml.outlier.StochasticOutlierSelection.ErrorTolerance]]:
- * The accepted error tolerance when computing the perplexity. When increasing this number, it
- * will sacrifice accuracy in return for reduced computational time.
- * (Default value: '''1e-20''')
- *
- * - [[org.apache.flink.ml.outlier.StochasticOutlierSelection.MaxIterations]]:
- * The maximum number of iterations to perform to constrain the computational time.
- * (Default value: '''5000''')
- */
-
-import breeze.linalg.functions.euclideanDistance
-import breeze.linalg.{sum, DenseVector => BreezeDenseVector, Vector => BreezeVector}
-import org.apache.flink.api.common.operators.Order
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.utils._
-import org.apache.flink.ml.common.{LabeledVector, Parameter, ParameterMap, WithParameters}
-import org.apache.flink.ml.math.Breeze._
-import org.apache.flink.ml.math.{BreezeVectorConverter, Vector}
-import org.apache.flink.ml.pipeline.{TransformDataSetOperation, Transformer}
-
-import scala.language.implicitConversions
-import scala.reflect.ClassTag
-
-class StochasticOutlierSelection extends Transformer[StochasticOutlierSelection] {
-
- import StochasticOutlierSelection._
-
-
- /** Sets the perplexity of the outlier selection algorithm, can be seen as the k of kNN
- * For more information, please read the Stochastic Outlier Selection algorithm technical paper.
- *
- * @param perplexity the perplexity of the affinity fit
- * @return
- */
- def setPerplexity(perplexity: Double): StochasticOutlierSelection = {
- require(perplexity >= 1, "Perplexity must be at least one.")
- parameters.add(Perplexity, perplexity)
- this
- }
-
- /** The accepted error tolerance to reduce computational time when approximating the affinity.
- *
- * @param errorTolerance the accepted error tolerance with respect to the affinity
- * @return
- */
- def setErrorTolerance(errorTolerance: Double): StochasticOutlierSelection = {
- require(errorTolerance >= 0, "Error tolerance cannot be negative.")
- parameters.add(ErrorTolerance, errorTolerance)
- this
- }
-
- /** The maximum number of iterations to approximate the affinity of the algorithm.
- *
- * @param maxIterations the maximum number of iterations.
- * @return
- */
- def setMaxIterations(maxIterations: Int): StochasticOutlierSelection = {
- require(maxIterations > 0, "Maximum iterations must be positive.")
- parameters.add(MaxIterations, maxIterations)
- this
- }
-
-}
-
-object StochasticOutlierSelection extends WithParameters {
-
- // ========================================= Parameters ==========================================
- case object Perplexity extends Parameter[Double] {
- val defaultValue: Option[Double] = Some(30)
- }
-
- case object ErrorTolerance extends Parameter[Double] {
- val defaultValue: Option[Double] = Some(1e-20)
- }
-
- case object MaxIterations extends Parameter[Int] {
- val defaultValue: Option[Int] = Some(5000)
- }
-
- // ==================================== Factory methods ==========================================
-
- def apply(): StochasticOutlierSelection = {
- new StochasticOutlierSelection()
- }
-
- // ===================================== Operations ==============================================
- case class BreezeLabeledVector(idx: Int, data: BreezeVector[Double])
-
- implicit val transformLabeledVectors = {
-
- new TransformDataSetOperation[StochasticOutlierSelection, LabeledVector, (Int, Double)] {
-
-
- /** Overrides the method of the parent class and applies the stochastic outlier selection
- * algorithm.
- *
- * @param instance Instance of the class
- * @param transformParameters The user defined parameters of the algorithm
- * @param input A data set which consists of all the LabeledVectors, which should have an
- * index or unique integer label as vector.
- * @return The outlierness of the vectors compared to each other
- */
- override def transformDataSet(instance: StochasticOutlierSelection,
- transformParameters: ParameterMap,
- input: DataSet[LabeledVector]): DataSet[(Int, Double)] = {
-
- val resultingParameters = instance.parameters ++ transformParameters
-
- val vectorsWithIndex = input.map(labeledVector => {
- BreezeLabeledVector(labeledVector.label.toInt, labeledVector.vector.asBreeze)
- })
-
- // Don't map back to a labeled-vector since the output of the algorithm is
- // a single double instead of vector
- outlierSelection(vectorsWithIndex, resultingParameters)
- }
- }
- }
-
- /** [[TransformDataSetOperation]] applies the stochastic outlier selection algorithm on a
- * [[Vector]] which will transform the high-dimensional input to a single Double output.
- *
- * @tparam T Type of the input and output data which has to be a subtype of [[Vector]]
- * @return [[TransformDataSetOperation]] a single double which represents the oulierness of
- * the input vectors, where the output is in [0, 1]
- */
- implicit def transformVectors[T <: Vector : BreezeVectorConverter : TypeInformation : ClassTag]
- = {
- new TransformDataSetOperation[StochasticOutlierSelection, T, Double] {
- override def transformDataSet(instance: StochasticOutlierSelection,
- transformParameters: ParameterMap,
- input: DataSet[T]): DataSet[Double] = {
-
- val resultingParameters = instance.parameters ++ transformParameters
-
- // Map to the right format
- val vectorsWithIndex = input.zipWithUniqueId.map((vector: (Long, T)) => {
- BreezeLabeledVector(vector._1.toInt, vector._2.asBreeze)
- })
-
- outlierSelection(vectorsWithIndex, resultingParameters).map(_._2)
- }
- }
- }
-
- /** Internal entry point which will execute the different stages of the algorithm using a single
- * interface
- *
- * @param inputVectors Input vectors on which the stochastic outlier selection algorithm
- * will be applied which should be the index or a unique integer value
- * @param transformParameters The user defined parameters of the algorithm
- * @return The outlierness of the vectors compared to each other
- */
- private def outlierSelection(inputVectors: DataSet[BreezeLabeledVector],
- transformParameters: ParameterMap): DataSet[(Int, Double)] = {
- val dissimilarityVectors = computeDissimilarityVectors(inputVectors)
- val affinityVectors = computeAffinity(dissimilarityVectors, transformParameters)
- val bindingProbabilityVectors = computeBindingProbabilities(affinityVectors)
- val outlierProbability = computeOutlierProbability(bindingProbabilityVectors)
-
- outlierProbability
- }
-
- /** Compute pair-wise distance from each vector, to all other vectors.
- *
- * @param inputVectors The input vectors, will compare the vector to all other vectors based
- * on an distance method.
- * @return Returns new set of [[BreezeLabeledVector]] with dissimilarity vector
- */
- def computeDissimilarityVectors(inputVectors: DataSet[BreezeLabeledVector]):
- DataSet[BreezeLabeledVector] =
- inputVectors.cross(inputVectors) {
- (a, b) => (a.idx, b.idx, euclideanDistance(a.data, b.data))
- }.filter(dist => dist._1 != dist._2) // Filter out the diagonal, this contains no information.
- .groupBy(0)
- .sortGroup(1, Order.ASCENDING)
- .reduceGroup {
- distancesIterator => {
- val distances = distancesIterator.toList
- val distanceVector = distances.map(_._3).toArray
-
- BreezeLabeledVector(distances.head._1, BreezeDenseVector(distanceVector))
- }
- }
-
- /** Approximate the affinity by fitting a Gaussian-like function
- *
- * @param dissimilarityVectors The dissimilarity vectors which represents the distance to the
- * other vectors in the data set.
- * @param resultingParameters The user defined parameters of the algorithm
- * @return Returns new set of [[BreezeLabeledVector]] with dissimilarity vector
- */
- def computeAffinity(dissimilarityVectors: DataSet[BreezeLabeledVector],
- resultingParameters: ParameterMap): DataSet[BreezeLabeledVector] = {
- val logPerplexity = Math.log(resultingParameters(Perplexity))
- val maxIterations = resultingParameters(MaxIterations)
- val errorTolerance = resultingParameters(ErrorTolerance)
-
- dissimilarityVectors.map(vec => {
- val breezeVec = binarySearch(vec.data, logPerplexity, maxIterations, errorTolerance)
- BreezeLabeledVector(vec.idx, breezeVec)
- })
- }
-
- /** Normalizes the input vectors so each row sums up to one.
- *
- * @param affinityVectors The affinity vectors which is the quantification of the relationship
- * between the original vectors.
- * @return Returns new set of [[BreezeLabeledVector]] with represents the binding
- * probabilities, which is in fact the affinity where each row sums up to one.
- */
- def computeBindingProbabilities(affinityVectors: DataSet[BreezeLabeledVector]):
- DataSet[BreezeLabeledVector] =
- affinityVectors.map(vec => BreezeLabeledVector(vec.idx, vec.data :/ sum(vec.data)))
-
- /** Compute the final outlier probability by taking the product of the column.
- *
- * @param bindingProbabilityVectors The binding probability vectors where the binding
- * probability is based on the affinity and represents the
- * probability of a vector binding with another vector.
- * @return Returns a single double which represents the final outlierness of the input vector.
- */
- def computeOutlierProbability(bindingProbabilityVectors: DataSet[BreezeLabeledVector]):
- DataSet[(Int, Double)] = bindingProbabilityVectors
- .flatMap(vec => vec.data.toArray.zipWithIndex.map(pair => {
-
- // The DistanceMatrix removed the diagonal, but we need to compute the product
- // of the column, so we need to correct the offset.
- val columnIndex = if (pair._2 >= vec.idx) {
- 1
- } else {
- 0
- }
-
- (columnIndex + pair._2, pair._1)
- })).groupBy(0).reduceGroup {
- probabilities => {
- var rowNumber = -1
- var outlierProbability = 1.0
- for (probability <- probabilities) {
- rowNumber = probability._1
- outlierProbability = outlierProbability * (1.0 - probability._2)
- }
-
- (rowNumber, outlierProbability)
- }
- }
-
- /** Performs a binary search to get affinities in such a way that each conditional Gaussian has
- * the same perplexity.
- *
- * @param dissimilarityVector The input dissimilarity vector which represents the current
- * vector distance to the other vectors in the data set
- * @param logPerplexity The log of the perplexity, which represents the probability of having
- * affinity with another vector.
- * @param maxIterations The maximum iterations to limit the computational time.
- * @param tolerance The allowed tolerance to sacrifice precision for decreased computational
- * time.
- * @param beta: The current beta
- * @param betaMin The lower bound of beta
- * @param betaMax The upper bound of beta
- * @param iteration The current iteration
- * @return Returns the affinity vector of the input vector.
- */
- def binarySearch(
- dissimilarityVector: BreezeVector[Double],
- logPerplexity: Double,
- maxIterations: Int,
- tolerance: Double,
- beta: Double = 1.0,
- betaMin: Double = Double.NegativeInfinity,
- betaMax: Double = Double.PositiveInfinity,
- iteration: Int = 0)
- : BreezeVector[Double] = {
-
- val newAffinity = dissimilarityVector.map(d => Math.exp(-d * beta))
- val sumA = sum(newAffinity)
- val hCurr = Math.log(sumA) + beta * sum(dissimilarityVector :* newAffinity) / sumA
- val hDiff = hCurr - logPerplexity
-
- if (iteration < maxIterations && Math.abs(hDiff) > tolerance) {
- // Compute the Gaussian kernel and entropy for the current precision
- val (newBeta, newBetaMin, newBetaMax) = if (hDiff.isNaN) {
- (beta / 10.0, betaMin, betaMax) // Reduce beta to get it in range
- } else {
- if (hDiff > 0) {
- val newBeta =
- if (betaMax == Double.PositiveInfinity || betaMax == Double.NegativeInfinity) {
- beta * 2.0
- } else {
- (beta + betaMax) / 2.0
- }
-
- (newBeta, beta, betaMax)
- } else {
- val newBeta =
- if (betaMin == Double.PositiveInfinity || betaMin == Double.NegativeInfinity) {
- beta / 2.0
- } else {
- (beta + betaMin) / 2.0
- }
-
- (newBeta, betaMin, beta)
- }
- }
-
- binarySearch(dissimilarityVector,
- logPerplexity,
- maxIterations,
- tolerance,
- newBeta,
- newBetaMin,
- newBetaMax,
- iteration + 1)
- }
- else {
- newAffinity
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala
deleted file mode 100644
index 554e155..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/package.scala
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink
-
-import org.apache.flink.api.common.functions.{RichFilterFunction, RichMapFunction}
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.java.operators.DataSink
-import org.apache.flink.api.scala.{DataSet, ExecutionEnvironment}
-import org.apache.flink.configuration.Configuration
-import org.apache.flink.ml.common.LabeledVector
-
-import scala.reflect.ClassTag
-
-package object ml {
-
- /** Pimp my [[ExecutionEnvironment]] to directly support `readLibSVM`
- *
- * @param executionEnvironment
- */
- implicit class RichExecutionEnvironment(executionEnvironment: ExecutionEnvironment) {
- def readLibSVM(path: String): DataSet[LabeledVector] = {
- MLUtils.readLibSVM(executionEnvironment, path)
- }
- }
-
- /** Pimp my [[DataSet]] to directly support `writeAsLibSVM`
- *
- * @param dataSet
- */
- implicit class RichLabeledDataSet(dataSet: DataSet[LabeledVector]) {
- def writeAsLibSVM(path: String): DataSink[String] = {
- MLUtils.writeLibSVM(path, dataSet)
- }
- }
-
- implicit class RichDataSet[T](dataSet: DataSet[T]) {
- def mapWithBcVariable[B, O: TypeInformation: ClassTag](
- broadcastVariable: DataSet[B])(
- fun: (T, B) => O)
- : DataSet[O] = {
- dataSet.map(new BroadcastSingleElementMapper[T, B, O](dataSet.clean(fun)))
- .withBroadcastSet(broadcastVariable, "broadcastVariable")
- }
-
- def filterWithBcVariable[B, O](broadcastVariable: DataSet[B])(fun: (T, B) => Boolean)
- : DataSet[T] = {
- dataSet.filter(new BroadcastSingleElementFilter[T, B](dataSet.clean(fun)))
- .withBroadcastSet(broadcastVariable, "broadcastVariable")
- }
-
- def mapWithBcVariableIteration[B, O: TypeInformation: ClassTag](
- broadcastVariable: DataSet[B])(fun: (T, B, Int) => O)
- : DataSet[O] = {
- dataSet.map(new BroadcastSingleElementMapperWithIteration[T, B, O](dataSet.clean(fun)))
- .withBroadcastSet(broadcastVariable, "broadcastVariable")
- }
- }
-
- private class BroadcastSingleElementMapper[T, B, O](
- fun: (T, B) => O)
- extends RichMapFunction[T, O] {
- var broadcastVariable: B = _
-
- @throws(classOf[Exception])
- override def open(configuration: Configuration): Unit = {
- broadcastVariable = getRuntimeContext.getBroadcastVariable[B]("broadcastVariable").get(0)
- }
-
- override def map(value: T): O = {
- fun(value, broadcastVariable)
- }
- }
-
- private class BroadcastSingleElementMapperWithIteration[T, B, O](
- fun: (T, B, Int) => O)
- extends RichMapFunction[T, O] {
- var broadcastVariable: B = _
-
- @throws(classOf[Exception])
- override def open(configuration: Configuration): Unit = {
- broadcastVariable = getRuntimeContext.getBroadcastVariable[B]("broadcastVariable").get(0)
- }
-
- override def map(value: T): O = {
- fun(value, broadcastVariable, getIterationRuntimeContext.getSuperstepNumber)
- }
- }
-
- private class BroadcastSingleElementFilter[T, B](
- fun: (T, B) => Boolean)
- extends RichFilterFunction[T] {
- var broadcastVariable: B = _
-
- @throws(classOf[Exception])
- override def open(configuration: Configuration): Unit = {
- broadcastVariable = getRuntimeContext.getBroadcastVariable[B]("broadcastVariable").get(0)
- }
-
- override def filter(value: T): Boolean = {
- fun(value, broadcastVariable)
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/ChainedPredictor.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/ChainedPredictor.scala
deleted file mode 100644
index bf5a8b2..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/ChainedPredictor.scala
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.pipeline
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.ml.common.ParameterMap
-
-/** [[Predictor]] which represents a pipeline of possibly multiple [[Transformer]] and a trailing
- * [[Predictor]].
- *
- * The [[ChainedPredictor]] can be used as a regular [[Predictor]]. Upon calling the fit method,
- * the input data is piped through all preceding [[Transformer]] in the pipeline and the resulting
- * data is given to the trailing [[Predictor]]. The same holds true for the predict operation.
- *
- * The pipeline mechanism has been inspired by scikit-learn
- *
- * @param transformer Preceding [[Transformer]] of the pipeline
- * @param predictor Trailing [[Predictor]] of the pipeline
- * @tparam T Type of the preceding [[Transformer]]
- * @tparam P Type of the trailing [[Predictor]]
- */
-case class ChainedPredictor[T <: Transformer[T], P <: Predictor[P]](transformer: T, predictor: P)
- extends Predictor[ChainedPredictor[T, P]]{}
-
-object ChainedPredictor{
-
- /** [[PredictDataSetOperation]] for the [[ChainedPredictor]].
- *
- * The [[PredictDataSetOperation]] requires the [[TransformDataSetOperation]] of the preceding
- * [[Transformer]] and the [[PredictDataSetOperation]] of the trailing [[Predictor]]. Upon
- * calling predict, the testing data is first transformed by the preceding [[Transformer]] and
- * the result is then used to calculate the prediction via the trailing [[Predictor]].
- *
- * @param transformOperation [[TransformDataSetOperation]] for the preceding [[Transformer]]
- * @param predictOperation [[PredictDataSetOperation]] for the trailing [[Predictor]]
- * @tparam T Type of the preceding [[Transformer]]
- * @tparam P Type of the trailing [[Predictor]]
- * @tparam Testing Type of the testing data
- * @tparam Intermediate Type of the intermediate data produced by the preceding [[Transformer]]
- * @tparam Prediction Type of the predicted data generated by the trailing [[Predictor]]
- * @return
- */
- implicit def chainedPredictOperation[
- T <: Transformer[T],
- P <: Predictor[P],
- Testing,
- Intermediate,
- Prediction](
- implicit transformOperation: TransformDataSetOperation[T, Testing, Intermediate],
- predictOperation: PredictDataSetOperation[P, Intermediate, Prediction])
- : PredictDataSetOperation[ChainedPredictor[T, P], Testing, Prediction] = {
-
- new PredictDataSetOperation[ChainedPredictor[T, P], Testing, Prediction] {
- override def predictDataSet(
- instance: ChainedPredictor[T, P],
- predictParameters: ParameterMap,
- input: DataSet[Testing])
- : DataSet[Prediction] = {
-
- val testing = instance.transformer.transform(input, predictParameters)
- instance.predictor.predict(testing, predictParameters)
- }
- }
- }
-
- /** [[FitOperation]] for the [[ChainedPredictor]].
- *
- * The [[FitOperation]] requires the [[FitOperation]] and the [[TransformDataSetOperation]] of
- * the preceding [[Transformer]] as well as the [[FitOperation]] of the trailing [[Predictor]].
- * Upon calling fit, the preceding [[Transformer]] is first fitted to the training data.
- * The training data is then transformed by the fitted [[Transformer]]. The transformed data
- * is then used to fit the [[Predictor]].
- *
- * @param fitOperation [[FitOperation]] of the preceding [[Transformer]]
- * @param transformOperation [[TransformDataSetOperation]] of the preceding [[Transformer]]
- * @param predictorFitOperation [[PredictDataSetOperation]] of the trailing [[Predictor]]
- * @tparam L Type of the preceding [[Transformer]]
- * @tparam R Type of the trailing [[Predictor]]
- * @tparam I Type of the training data
- * @tparam T Type of the intermediate data
- * @return
- */
- implicit def chainedFitOperation[L <: Transformer[L], R <: Predictor[R], I, T](implicit
- fitOperation: FitOperation[L, I],
- transformOperation: TransformDataSetOperation[L, I, T],
- predictorFitOperation: FitOperation[R, T]): FitOperation[ChainedPredictor[L, R], I] = {
- new FitOperation[ChainedPredictor[L, R], I] {
- override def fit(
- instance: ChainedPredictor[L, R],
- fitParameters: ParameterMap,
- input: DataSet[I])
- : Unit = {
- instance.transformer.fit(input, fitParameters)
- val intermediateResult = instance.transformer.transform(input, fitParameters)
- instance.predictor.fit(intermediateResult, fitParameters)
- }
- }
- }
-
- implicit def chainedEvaluationOperation[
- T <: Transformer[T],
- P <: Predictor[P],
- Testing,
- Intermediate,
- PredictionValue](
- implicit transformOperation: TransformDataSetOperation[T, Testing, Intermediate],
- evaluateOperation: EvaluateDataSetOperation[P, Intermediate, PredictionValue],
- testingTypeInformation: TypeInformation[Testing],
- predictionValueTypeInformation: TypeInformation[PredictionValue])
- : EvaluateDataSetOperation[ChainedPredictor[T, P], Testing, PredictionValue] = {
- new EvaluateDataSetOperation[ChainedPredictor[T, P], Testing, PredictionValue] {
- override def evaluateDataSet(
- instance: ChainedPredictor[T, P],
- evaluateParameters: ParameterMap,
- testing: DataSet[Testing])
- : DataSet[(PredictionValue, PredictionValue)] = {
- val intermediate = instance.transformer.transform(testing, evaluateParameters)
- instance.predictor.evaluate(intermediate, evaluateParameters)
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/ChainedTransformer.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/ChainedTransformer.scala
deleted file mode 100644
index bdf917d..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/ChainedTransformer.scala
+++ /dev/null
@@ -1,110 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.pipeline
-
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.ml.common.ParameterMap
-
-/** [[Transformer]] which represents the chaining of two [[Transformer]].
- *
- * A [[ChainedTransformer]] can be treated as regular [[Transformer]]. Upon calling the fit or
- * transform operation, the data is piped through all [[Transformer]] of the pipeline.
- *
- * The pipeline mechanism has been inspired by scikit-learn
- *
- * @param left Left [[Transformer]] of the pipeline
- * @param right Right [[Transformer]] of the pipeline
- * @tparam L Type of the left [[Transformer]]
- * @tparam R Type of the right [[Transformer]]
- */
-case class ChainedTransformer[L <: Transformer[L], R <: Transformer[R]](left: L, right: R)
- extends Transformer[ChainedTransformer[L, R]] {
-}
-
-object ChainedTransformer{
-
- /** [[TransformDataSetOperation]] implementation for [[ChainedTransformer]].
- *
- * First the transform operation of the left [[Transformer]] is called with the input data. This
- * generates intermediate data which is fed to the right [[Transformer]]'s transform operation.
- *
- * @param transformOpLeft [[TransformDataSetOperation]] for the left [[Transformer]]
- * @param transformOpRight [[TransformDataSetOperation]] for the right [[Transformer]]
- * @tparam L Type of the left [[Transformer]]
- * @tparam R Type of the right [[Transformer]]
- * @tparam I Type of the input data
- * @tparam T Type of the intermediate output data
- * @tparam O Type of the output data
- * @return
- */
- implicit def chainedTransformOperation[
- L <: Transformer[L],
- R <: Transformer[R],
- I,
- T,
- O](implicit
- transformOpLeft: TransformDataSetOperation[L, I, T],
- transformOpRight: TransformDataSetOperation[R, T, O])
- : TransformDataSetOperation[ChainedTransformer[L,R], I, O] = {
-
- new TransformDataSetOperation[ChainedTransformer[L, R], I, O] {
- override def transformDataSet(
- chain: ChainedTransformer[L, R],
- transformParameters: ParameterMap,
- input: DataSet[I]): DataSet[O] = {
- val intermediateResult = transformOpLeft.transformDataSet(
- chain.left,
- transformParameters,
- input)
- transformOpRight.transformDataSet(chain.right, transformParameters, intermediateResult)
- }
- }
- }
-
- /** [[FitOperation]] implementation for [[ChainedTransformer]].
- *
- * First the fit operation of the left [[Transformer]] is called with the input data. Then
- * the data is transformed by this [[Transformer]] and the given to the fit operation of the
- * right [[Transformer]].
- *
- * @param leftFitOperation [[FitOperation]] for the left [[Transformer]]
- * @param leftTransformOperation [[TransformDataSetOperation]] for the left [[Transformer]]
- * @param rightFitOperation [[FitOperation]] for the right [[Transformer]]
- * @tparam L Type of the left [[Transformer]]
- * @tparam R Type of the right [[Transformer]]
- * @tparam I Type of the input data
- * @tparam T Type of the intermediate output data
- * @return
- */
- implicit def chainedFitOperation[L <: Transformer[L], R <: Transformer[R], I, T](implicit
- leftFitOperation: FitOperation[L, I],
- leftTransformOperation: TransformDataSetOperation[L, I, T],
- rightFitOperation: FitOperation[R, T]): FitOperation[ChainedTransformer[L, R], I] = {
- new FitOperation[ChainedTransformer[L, R], I] {
- override def fit(
- instance: ChainedTransformer[L, R],
- fitParameters: ParameterMap,
- input: DataSet[I]): Unit = {
- instance.left.fit(input, fitParameters)
- val intermediateResult = instance.left.transform(input, fitParameters)
- instance.right.fit(intermediateResult, fitParameters)
- }
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Estimator.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Estimator.scala
deleted file mode 100644
index ca7cb33..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Estimator.scala
+++ /dev/null
@@ -1,181 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.pipeline
-
-import scala.reflect.ClassTag
-import scala.reflect.runtime.universe._
-
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.ml.common.{FlinkMLTools, ParameterMap, WithParameters}
-
-/** Base trait for Flink's pipeline operators.
- *
- * An estimator can be fitted to input data. In order to do that the implementing class has
- * to provide an implementation of a [[FitOperation]] with the correct input type. In order to make
- * the [[FitOperation]] retrievable by the Scala compiler, the implementation should be placed
- * in the companion object of the implementing class.
- *
- * The pipeline mechanism has been inspired by scikit-learn
- *
- * @tparam Self
- */
-trait Estimator[Self] extends WithParameters {
- that: Self =>
-
- /** Fits the estimator to the given input data. The fitting logic is contained in the
- * [[FitOperation]]. The computed state will be stored in the implementing class.
- *
- * @param training Training data
- * @param fitParameters Additional parameters for the [[FitOperation]]
- * @param fitOperation [[FitOperation]] which encapsulates the algorithm logic
- * @tparam Training Type of the training data
- * @return
- */
- def fit[Training](
- training: DataSet[Training],
- fitParameters: ParameterMap = ParameterMap.Empty)(implicit
- fitOperation: FitOperation[Self, Training]): Unit = {
- FlinkMLTools.registerFlinkMLTypes(training.getExecutionEnvironment)
- fitOperation.fit(this, fitParameters, training)
- }
-}
-
-object Estimator{
-
- /** Fallback [[FitOperation]] type class implementation which is used if no other
- * [[FitOperation]] with the right input types could be found in the scope of the implementing
- * class. The fallback [[FitOperation]] makes the system fail in the pre-flight phase by
- * throwing a [[RuntimeException]] which states the reason for the failure. Usually the error
- * is a missing [[FitOperation]] implementation for the input types or the wrong chaining
- * of pipeline operators which have incompatible input/output types.
- *
- * @tparam Self Type of the pipeline operator
- * @tparam Training Type of training data
- * @return
- */
- implicit def fallbackFitOperation[
- Self: TypeTag,
- Training: TypeTag]
- : FitOperation[Self, Training] = {
- new FitOperation[Self, Training]{
- override def fit(
- instance: Self,
- fitParameters: ParameterMap,
- input: DataSet[Training])
- : Unit = {
- val self = typeOf[Self]
- val training = typeOf[Training]
-
- throw new RuntimeException("There is no FitOperation defined for " + self +
- " which trains on a DataSet[" + training + "]")
- }
- }
- }
-
- /** Fallback [[PredictDataSetOperation]] if a [[Predictor]] is called with a not supported input
- * data type. The fallback [[PredictDataSetOperation]] lets the system fail with a
- * [[RuntimeException]] stating which input and output data types were inferred but for which no
- * [[PredictDataSetOperation]] could be found.
- *
- * @tparam Self Type of the [[Predictor]]
- * @tparam Testing Type of the testing data
- * @return
- */
- implicit def fallbackPredictOperation[
- Self: TypeTag,
- Testing: TypeTag]
- : PredictDataSetOperation[Self, Testing, Any] = {
- new PredictDataSetOperation[Self, Testing, Any] {
- override def predictDataSet(
- instance: Self,
- predictParameters: ParameterMap,
- input: DataSet[Testing])
- : DataSet[Any] = {
- val self = typeOf[Self]
- val testing = typeOf[Testing]
-
- throw new RuntimeException("There is no PredictOperation defined for " + self +
- " which takes a DataSet[" + testing + "] as input.")
- }
- }
- }
-
- /** Fallback [[TransformDataSetOperation]] for [[Transformer]] which do not support the input or
- * output type with which they are called. This is usually the case if pipeline operators are
- * chained which have incompatible input/output types. In order to detect these failures, the
- * fallback [[TransformDataSetOperation]] throws a [[RuntimeException]] with the corresponding
- * input/output types. Consequently, a wrong pipeline will be detected at pre-flight phase of
- * Flink and thus prior to execution time.
- *
- * @tparam Self Type of the [[Transformer]] for which the [[TransformDataSetOperation]] is
- * defined
- * @tparam IN Input data type of the [[TransformDataSetOperation]]
- * @return
- */
- implicit def fallbackTransformOperation[
- Self: TypeTag,
- IN: TypeTag]
- : TransformDataSetOperation[Self, IN, Any] = {
- new TransformDataSetOperation[Self, IN, Any] {
- override def transformDataSet(
- instance: Self,
- transformParameters: ParameterMap,
- input: DataSet[IN])
- : DataSet[Any] = {
- val self = typeOf[Self]
- val in = typeOf[IN]
-
- throw new RuntimeException("There is no TransformOperation defined for " +
- self + " which takes a DataSet[" + in +
- "] as input.")
- }
- }
- }
-
- implicit def fallbackEvaluateOperation[
- Self: TypeTag,
- Testing: TypeTag]
- : EvaluateDataSetOperation[Self, Testing, Any] = {
- new EvaluateDataSetOperation[Self, Testing, Any] {
- override def evaluateDataSet(
- instance: Self,
- predictParameters: ParameterMap,
- input: DataSet[Testing])
- : DataSet[(Any, Any)] = {
- val self = typeOf[Self]
- val testing = typeOf[Testing]
-
- throw new RuntimeException("There is no PredictOperation defined for " + self +
- " which takes a DataSet[" + testing + "] as input.")
- }
- }
- }
-}
-
-/** Type class for the fit operation of an [[Estimator]].
- *
- * The [[FitOperation]] contains a self type parameter so that the Scala compiler looks into
- * the companion object of this class to find implicit values.
- *
- * @tparam Self Type of the [[Estimator]] subclass for which the [[FitOperation]] is defined
- * @tparam Training Type of the training data
- */
-trait FitOperation[Self, Training]{
- def fit(instance: Self, fitParameters: ParameterMap, input: DataSet[Training]): Unit
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala
deleted file mode 100644
index d0f3064..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Predictor.scala
+++ /dev/null
@@ -1,258 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.pipeline
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-
-import org.apache.flink.api.scala._
-import org.apache.flink.ml._
-import org.apache.flink.ml.common.{FlinkMLTools, ParameterMap, WithParameters}
-
-/** Predictor trait for Flink's pipeline operators.
- *
- * A [[Predictor]] calculates predictions for testing data based on the model it learned during
- * the fit operation (training phase). In order to do that, the implementing class has to provide
- * a [[FitOperation]] and a [[PredictDataSetOperation]] implementation for the correct types. The
- * implicit values should be put into the scope of the companion object of the implementing class
- * to make them retrievable for the Scala compiler.
- *
- * The pipeline mechanism has been inspired by scikit-learn
- *
- * @tparam Self Type of the implementing class
- */
-trait Predictor[Self] extends Estimator[Self] with WithParameters {
- that: Self =>
-
- /** Predict testing data according the learned model. The implementing class has to provide
- * a corresponding implementation of [[PredictDataSetOperation]] which contains the prediction
- * logic.
- *
- * @param testing Testing data which shall be predicted
- * @param predictParameters Additional parameters for the prediction
- * @param predictor [[PredictDataSetOperation]] which encapsulates the prediction logic
- * @tparam Testing Type of the testing data
- * @tparam Prediction Type of the prediction data
- * @return
- */
- def predict[Testing, Prediction](
- testing: DataSet[Testing],
- predictParameters: ParameterMap = ParameterMap.Empty)(implicit
- predictor: PredictDataSetOperation[Self, Testing, Prediction])
- : DataSet[Prediction] = {
- FlinkMLTools.registerFlinkMLTypes(testing.getExecutionEnvironment)
- predictor.predictDataSet(this, predictParameters, testing)
- }
-
- /** Evaluates the testing data by computing the prediction value and returning a pair of true
- * label value and prediction value. It is important that the implementation chooses a Testing
- * type from which it can extract the true label value.
- *
- * @param testing
- * @param evaluateParameters
- * @param evaluator
- * @tparam Testing
- * @tparam PredictionValue
- * @return
- */
- def evaluate[Testing, PredictionValue](
- testing: DataSet[Testing],
- evaluateParameters: ParameterMap = ParameterMap.Empty)(implicit
- evaluator: EvaluateDataSetOperation[Self, Testing, PredictionValue])
- : DataSet[(PredictionValue, PredictionValue)] = {
- FlinkMLTools.registerFlinkMLTypes(testing.getExecutionEnvironment)
- evaluator.evaluateDataSet(this, evaluateParameters, testing)
- }
-}
-
-object Predictor {
-
- /** Default [[PredictDataSetOperation]] which takes a [[PredictOperation]] to calculate a tuple
- * of testing element and its prediction value.
- *
- * Note: We have to put the TypeInformation implicit values for Testing and PredictionValue after
- * the PredictOperation implicit parameter. Otherwise, if it's defined as a context bound, then
- * the Scala compiler does not find the implicit [[PredictOperation]] value.
- *
- * @param predictOperation
- * @param testingTypeInformation
- * @param predictionValueTypeInformation
- * @tparam Instance
- * @tparam Model
- * @tparam Testing
- * @tparam PredictionValue
- * @return
- */
- implicit def defaultPredictDataSetOperation[
- Instance <: Estimator[Instance],
- Model,
- Testing,
- PredictionValue](
- implicit predictOperation: PredictOperation[Instance, Model, Testing, PredictionValue],
- testingTypeInformation: TypeInformation[Testing],
- predictionValueTypeInformation: TypeInformation[PredictionValue])
- : PredictDataSetOperation[Instance, Testing, (Testing, PredictionValue)] = {
- new PredictDataSetOperation[Instance, Testing, (Testing, PredictionValue)] {
- override def predictDataSet(
- instance: Instance,
- predictParameters: ParameterMap,
- input: DataSet[Testing])
- : DataSet[(Testing, PredictionValue)] = {
- val resultingParameters = instance.parameters ++ predictParameters
-
- val model = predictOperation.getModel(instance, resultingParameters)
-
- implicit val resultTypeInformation = createTypeInformation[(Testing, PredictionValue)]
-
- input.mapWithBcVariable(model){
- (element, model) => {
- (element, predictOperation.predict(element, model))
- }
- }
- }
- }
- }
-
- /** Default [[EvaluateDataSetOperation]] which takes a [[PredictOperation]] to calculate a tuple
- * of true label value and predicted label value.
- *
- * Note: We have to put the TypeInformation implicit values for Testing and PredictionValue after
- * the PredictOperation implicit parameter. Otherwise, if it's defined as a context bound, then
- * the Scala compiler does not find the implicit [[PredictOperation]] value.
- *
- * @param predictOperation
- * @param testingTypeInformation
- * @param predictionValueTypeInformation
- * @tparam Instance
- * @tparam Model
- * @tparam Testing
- * @tparam PredictionValue
- * @return
- */
- implicit def defaultEvaluateDataSetOperation[
- Instance <: Estimator[Instance],
- Model,
- Testing,
- PredictionValue](
- implicit predictOperation: PredictOperation[Instance, Model, Testing, PredictionValue],
- testingTypeInformation: TypeInformation[Testing],
- predictionValueTypeInformation: TypeInformation[PredictionValue])
- : EvaluateDataSetOperation[Instance, (Testing, PredictionValue), PredictionValue] = {
- new EvaluateDataSetOperation[Instance, (Testing, PredictionValue), PredictionValue] {
- override def evaluateDataSet(
- instance: Instance,
- evaluateParameters: ParameterMap,
- testing: DataSet[(Testing, PredictionValue)])
- : DataSet[(PredictionValue, PredictionValue)] = {
- val resultingParameters = instance.parameters ++ evaluateParameters
- val model = predictOperation.getModel(instance, resultingParameters)
-
- implicit val resultTypeInformation = createTypeInformation[(Testing, PredictionValue)]
-
- testing.mapWithBcVariable(model){
- (element, model) => {
- (element._2, predictOperation.predict(element._1, model))
- }
- }
- }
- }
- }
-}
-
-/** Type class for the predict operation of [[Predictor]]. This predict operation works on DataSets.
- *
- * [[Predictor]]s either have to implement this trait or the [[PredictOperation]] trait. The
- * implementation has to be made available as an implicit value or function in the scope of
- * their companion objects.
- *
- * The first type parameter is the type of the implementing [[Predictor]] class so that the Scala
- * compiler includes the companion object of this class in the search scope for the implicit
- * values.
- *
- * @tparam Self Type of [[Predictor]] implementing class
- * @tparam Testing Type of testing data
- * @tparam Prediction Type of predicted data
- */
-trait PredictDataSetOperation[Self, Testing, Prediction] extends Serializable{
-
- /** Calculates the predictions for all elements in the [[DataSet]] input
- *
- * @param instance The Predictor instance that we will use to make the predictions
- * @param predictParameters The parameters for the prediction
- * @param input The DataSet containing the unlabeled examples
- * @return
- */
- def predictDataSet(
- instance: Self,
- predictParameters: ParameterMap,
- input: DataSet[Testing])
- : DataSet[Prediction]
-}
-
-/** Type class for predict operation. It takes an element and the model and then computes the
- * prediction value for this element.
- *
- * It is sufficient for a [[Predictor]] to only implement this trait to support the evaluate and
- * predict method.
- *
- * @tparam Instance The concrete type of the [[Predictor]] that we will use for predictions
- * @tparam Model The representation of the predictive model for the algorithm, for example a
- * Vector of weights
- * @tparam Testing The type of the example that we will use to make the predictions (input)
- * @tparam Prediction The type of the label that the prediction operation will produce (output)
- *
- */
-trait PredictOperation[Instance, Model, Testing, Prediction] extends Serializable{
-
- /** Defines how to retrieve the model of the type for which this operation was defined
- *
- * @param instance The Predictor instance that we will use to make the predictions
- * @param predictParameters The parameters for the prediction
- * @return A DataSet with the model representation as its only element
- */
- def getModel(instance: Instance, predictParameters: ParameterMap): DataSet[Model]
-
- /** Calculates the prediction for a single element given the model of the [[Predictor]].
- *
- * @param value The unlabeled example on which we make the prediction
- * @param model The model representation of the prediction algorithm
- * @return A label for the provided example of type [[Prediction]]
- */
- def predict(value: Testing, model: Model):
- Prediction
-}
-
-/** Type class for the evaluate operation of [[Predictor]]. This evaluate operation works on
- * DataSets.
- *
- * It takes a [[DataSet]] of some type. For each element of this [[DataSet]] the evaluate method
- * computes the prediction value and returns a tuple of true label value and prediction value.
- *
- * @tparam Instance The concrete type of the Predictor instance that we will use to make the
- * predictions
- * @tparam Testing The type of the example that we will use to make the predictions (input)
- * @tparam Prediction The type of the label that the prediction operation will produce (output)
- *
- */
-trait EvaluateDataSetOperation[Instance, Testing, Prediction] extends Serializable{
- def evaluateDataSet(
- instance: Instance,
- evaluateParameters: ParameterMap,
- testing: DataSet[Testing])
- : DataSet[(Prediction, Prediction)]
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Transformer.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Transformer.scala
deleted file mode 100644
index 4b44127..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/pipeline/Transformer.scala
+++ /dev/null
@@ -1,164 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.pipeline
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.ml._
-import org.apache.flink.ml.common.{FlinkMLTools, ParameterMap, WithParameters}
-
-import scala.reflect.ClassTag
-
-/** Transformer trait for Flink's pipeline operators.
- *
- * A Transformer transforms a [[DataSet]] of an input type into a [[DataSet]] of an output type.
- * Furthermore, a [[Transformer]] is also an [[Estimator]], because some transformations depend
- * on the training data. In order to do that the implementing class has to provide a
- * [[TransformDataSetOperation]] and [[FitOperation]] implementation. The Scala compiler finds
- * these implicit values if it is put in the scope of the companion object of the implementing
- * class.
- *
- * [[Transformer]] can be chained with other [[Transformer]] and [[Predictor]] to create
- * pipelines. These pipelines can consist of an arbitrary number of [[Transformer]] and at most
- * one trailing [[Predictor]].
- *
- * The pipeline mechanism has been inspired by scikit-learn
- *
- * @tparam Self
- */
-trait Transformer[Self <: Transformer[Self]]
- extends Estimator[Self]
- with WithParameters
- with Serializable {
- that: Self =>
-
- /** Transform operation which transforms an input [[DataSet]] of type I into an output [[DataSet]]
- * of type O. The actual transform operation is implemented within the
- * [[TransformDataSetOperation]].
- *
- * @param input Input [[DataSet]] of type I
- * @param transformParameters Additional parameters for the [[TransformDataSetOperation]]
- * @param transformOperation [[TransformDataSetOperation]] which encapsulates the algorithm's
- * logic
- * @tparam Input Input data type
- * @tparam Output Output data type
- * @return
- */
- def transform[Input, Output](
- input: DataSet[Input],
- transformParameters: ParameterMap = ParameterMap.Empty)
- (implicit transformOperation: TransformDataSetOperation[Self, Input, Output])
- : DataSet[Output] = {
- FlinkMLTools.registerFlinkMLTypes(input.getExecutionEnvironment)
- transformOperation.transformDataSet(that, transformParameters, input)
- }
-
- /** Chains two [[Transformer]] to form a [[ChainedTransformer]].
- *
- * @param transformer Right side transformer of the resulting pipeline
- * @tparam T Type of the [[Transformer]]
- * @return
- */
- def chainTransformer[T <: Transformer[T]](transformer: T): ChainedTransformer[Self, T] = {
- ChainedTransformer(this, transformer)
- }
-
- /** Chains a [[Transformer]] with a [[Predictor]] to form a [[ChainedPredictor]].
- *
- * @param predictor Trailing [[Predictor]] of the resulting pipeline
- * @tparam P Type of the [[Predictor]]
- * @return
- */
- def chainPredictor[P <: Predictor[P]](predictor: P): ChainedPredictor[Self, P] = {
- ChainedPredictor(this, predictor)
- }
-}
-
-object Transformer{
- implicit def defaultTransformDataSetOperation[
- Instance <: Estimator[Instance],
- Model,
- Input,
- Output](
- implicit transformOperation: TransformOperation[Instance, Model, Input, Output],
- outputTypeInformation: TypeInformation[Output],
- outputClassTag: ClassTag[Output])
- : TransformDataSetOperation[Instance, Input, Output] = {
- new TransformDataSetOperation[Instance, Input, Output] {
- override def transformDataSet(
- instance: Instance,
- transformParameters: ParameterMap,
- input: DataSet[Input])
- : DataSet[Output] = {
- val resultingParameters = instance.parameters ++ transformParameters
- val model = transformOperation.getModel(instance, resultingParameters)
-
- input.mapWithBcVariable(model){
- (element, model) => transformOperation.transform(element, model)
- }
- }
- }
- }
-}
-
-/** Type class for a transform operation of [[Transformer]]. This works on [[DataSet]] of elements.
- *
- * The [[TransformDataSetOperation]] contains a self type parameter so that the Scala compiler
- * looks into the companion object of this class to find implicit values.
- *
- * @tparam Instance Type of the [[Transformer]] for which the [[TransformDataSetOperation]] is
- * defined
- * @tparam Input Input data type
- * @tparam Output Output data type
- */
-trait TransformDataSetOperation[Instance, Input, Output] extends Serializable{
- def transformDataSet(
- instance: Instance,
- transformParameters: ParameterMap,
- input: DataSet[Input])
- : DataSet[Output]
-}
-
-/** Type class for a transform operation which works on a single element and the corresponding model
- * of the [[Transformer]].
- *
- * @tparam Instance
- * @tparam Model
- * @tparam Input
- * @tparam Output
- */
-trait TransformOperation[Instance, Model, Input, Output] extends Serializable{
-
- /** Retrieves the model of the [[Transformer]] for which this operation has been defined.
- *
- * @param instance
- * @param transformParameters
- * @return
- */
- def getModel(instance: Instance, transformParameters: ParameterMap): DataSet[Model]
-
- /** Transforms a single element with respect to the model associated with the respective
- * [[Transformer]]
- *
- * @param element
- * @param model
- * @return
- */
- def transform(element: Input, model: Model): Output
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/MinMaxScaler.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/MinMaxScaler.scala
deleted file mode 100644
index b37748f..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/MinMaxScaler.scala
+++ /dev/null
@@ -1,265 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.flink.ml.preprocessing
-
-import breeze.linalg
-import breeze.linalg.{max, min}
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala._
-import org.apache.flink.ml._
-import org.apache.flink.ml.common.{LabeledVector, Parameter, ParameterMap}
-import org.apache.flink.ml.math.Breeze._
-import org.apache.flink.ml.math.{BreezeVectorConverter, Vector}
-import org.apache.flink.ml.pipeline.{TransformDataSetOperation, FitOperation,
-Transformer}
-import org.apache.flink.ml.preprocessing.MinMaxScaler.{Max, Min}
-
-import scala.reflect.ClassTag
-
-/** Scales observations, so that all features are in a user-specified range.
- * By default for [[MinMaxScaler]] transformer range = [0,1].
- *
- * This transformer takes a subtype of [[Vector]] of values and maps it to a
- * scaled subtype of [[Vector]] such that each feature lies between a user-specified range.
- *
- * This transformer can be prepended to all [[Transformer]] and
- * [[org.apache.flink.ml.pipeline.Predictor]] implementations which expect as input a subtype
- * of [[Vector]] or a [[LabeledVector]].
- *
- * @example
- * {{{
- * val trainingDS: DataSet[Vector] = env.fromCollection(data)
- * val transformer = MinMaxScaler().setMin(-1.0)
- *
- * transformer.fit(trainingDS)
- * val transformedDS = transformer.transform(trainingDS)
- * }}}
- *
- * =Parameters=
- *
- * - [[Min]]: The minimum value of the range of the transformed data set; by default equal to 0
- * - [[Max]]: The maximum value of the range of the transformed data set; by default
- * equal to 1
- */
-class MinMaxScaler extends Transformer[MinMaxScaler] {
-
- private [preprocessing] var metricsOption: Option[
- DataSet[(linalg.Vector[Double], linalg.Vector[Double])]
- ] = None
-
- /** Sets the minimum for the range of the transformed data
- *
- * @param min the user-specified minimum value.
- * @return the MinMaxScaler instance with its minimum value set to the user-specified value.
- */
- def setMin(min: Double): MinMaxScaler = {
- parameters.add(Min, min)
- this
- }
-
- /** Sets the maximum for the range of the transformed data
- *
- * @param max the user-specified maximum value.
- * @return the MinMaxScaler instance with its minimum value set to the user-specified value.
- */
- def setMax(max: Double): MinMaxScaler = {
- parameters.add(Max, max)
- this
- }
-}
-
-object MinMaxScaler {
-
- // ====================================== Parameters =============================================
-
- case object Min extends Parameter[Double] {
- override val defaultValue: Option[Double] = Some(0.0)
- }
-
- case object Max extends Parameter[Double] {
- override val defaultValue: Option[Double] = Some(1.0)
- }
-
- // ==================================== Factory methods ==========================================
-
- def apply(): MinMaxScaler = {
- new MinMaxScaler()
- }
-
- // ====================================== Operations =============================================
-
- /** Trains the [[MinMaxScaler]] by learning the minimum and maximum of each feature of the
- * training data. These values are used in the transform step to transform the given input data.
- *
- * @tparam T Input data type which is a subtype of [[Vector]]
- * @return [[FitOperation]] training the [[MinMaxScaler]] on subtypes of [[Vector]]
- */
- implicit def fitVectorMinMaxScaler[T <: Vector] = new FitOperation[MinMaxScaler, T] {
- override def fit(instance: MinMaxScaler, fitParameters: ParameterMap, input: DataSet[T])
- : Unit = {
- val metrics = extractFeatureMinMaxVectors(input)
-
- instance.metricsOption = Some(metrics)
- }
- }
-
- /** Trains the [[MinMaxScaler]] by learning the minimum and maximum of the features of the
- * training data which is of type [[LabeledVector]]. The minimum and maximum are used to
- * transform the given input data.
- *
- */
- implicit val fitLabeledVectorMinMaxScaler = {
- new FitOperation[MinMaxScaler, LabeledVector] {
- override def fit(
- instance: MinMaxScaler,
- fitParameters: ParameterMap,
- input: DataSet[LabeledVector])
- : Unit = {
- val vectorDS = input.map(_.vector)
- val metrics = extractFeatureMinMaxVectors(vectorDS)
-
- instance.metricsOption = Some(metrics)
- }
- }
- }
-
- /** Calculates in one pass over the data the features' minimum and maximum values.
- *
- * @param dataSet The data set for which we want to calculate the minimum and maximum values.
- * @return DataSet containing a single tuple of two vectors (minVector, maxVector).
- * The first vector represents the minimum values vector and the second is the maximum
- * values vector.
- */
- private def extractFeatureMinMaxVectors[T <: Vector](dataSet: DataSet[T])
- : DataSet[(linalg.Vector[Double], linalg.Vector[Double])] = {
-
- val minMax = dataSet.map {
- v: T => (v.asBreeze, v.asBreeze)
- }.reduce {
- (minMax1, minMax2) => {
-
- val tempMinimum = min(minMax1._1, minMax2._1)
- val tempMaximum = max(minMax1._2, minMax2._2)
-
- (tempMinimum, tempMaximum)
- }
- }
- minMax
- }
-
- /** [[TransformDataSetOperation]] which scales input data of subtype of [[Vector]] with respect to
- * the calculated minimum and maximum of the training data. The minimum and maximum
- * values of the resulting data is configurable.
- *
- * @tparam T Type of the input and output data which has to be a subtype of [[Vector]]
- * @return [[TransformDataSetOperation]] scaling subtypes of [[Vector]] such that the feature
- * values are in the configured range
- */
- implicit def transformVectors[T <: Vector : BreezeVectorConverter : TypeInformation : ClassTag]
- = {
- new TransformDataSetOperation[MinMaxScaler, T, T] {
- override def transformDataSet(
- instance: MinMaxScaler,
- transformParameters: ParameterMap,
- input: DataSet[T])
- : DataSet[T] = {
-
- val resultingParameters = instance.parameters ++ transformParameters
- val min = resultingParameters(Min)
- val max = resultingParameters(Max)
-
- instance.metricsOption match {
- case Some(metrics) => {
- input.mapWithBcVariable(metrics) {
- (vector, metrics) => {
- val (broadcastMin, broadcastMax) = metrics
- scaleVector(vector, broadcastMin, broadcastMax, min, max)
- }
- }
- }
-
- case None =>
- throw new RuntimeException("The MinMaxScaler has not been fitted to the data. " +
- "This is necessary to estimate the minimum and maximum of the data.")
- }
- }
- }
- }
-
- implicit val transformLabeledVectors = {
- new TransformDataSetOperation[MinMaxScaler, LabeledVector, LabeledVector] {
- override def transformDataSet(instance: MinMaxScaler,
- transformParameters: ParameterMap,
- input: DataSet[LabeledVector]): DataSet[LabeledVector] = {
- val resultingParameters = instance.parameters ++ transformParameters
- val min = resultingParameters(Min)
- val max = resultingParameters(Max)
-
- instance.metricsOption match {
- case Some(metrics) => {
- input.mapWithBcVariable(metrics) {
- (labeledVector, metrics) => {
- val (broadcastMin, broadcastMax) = metrics
- val LabeledVector(label, vector) = labeledVector
-
- LabeledVector(label, scaleVector(vector, broadcastMin, broadcastMax, min, max))
- }
- }
- }
-
- case None =>
- throw new RuntimeException("The MinMaxScaler has not been fitted to the data. " +
- "This is necessary to estimate the minimum and maximum of the data.")
- }
- }
- }
- }
-
- /** Scales a vector such that it's features lie in the range [min, max]
- *
- * @param vector Vector to scale
- * @param broadcastMin Vector containing for each feature the minimal value in the training set
- * @param broadcastMax Vector containing for each feature the maximal value in the training set
- * @param min Minimal value of range
- * @param max Maximal value of range
- * @tparam T Type of [[Vector]]
- * @return Scaled feature vector
- */
- private def scaleVector[T <: Vector: BreezeVectorConverter](
- vector: T,
- broadcastMin: linalg.Vector[Double],
- broadcastMax: linalg.Vector[Double],
- min: Double,
- max: Double)
- : T = {
- var myVector = vector.asBreeze
-
- //handle the case where a feature takes only one value
- val rangePerFeature = (broadcastMax - broadcastMin)
- for (i <- 0 until rangePerFeature.size) {
- if (rangePerFeature(i) == 0.0) {
- rangePerFeature(i)= 1.0
- }
- }
-
- myVector -= broadcastMin
- myVector :/= rangePerFeature
- myVector = (myVector :* (max - min)) + min
- myVector.fromBreeze
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/PolynomialFeatures.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/PolynomialFeatures.scala
deleted file mode 100644
index 977428d..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/PolynomialFeatures.scala
+++ /dev/null
@@ -1,209 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.preprocessing
-
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala.{DataSet, _}
-import org.apache.flink.ml.common.{LabeledVector, Parameter, ParameterMap}
-import org.apache.flink.ml.math.{Vector, VectorBuilder}
-import org.apache.flink.ml.pipeline.{FitOperation, TransformDataSetOperation, Transformer}
-import org.apache.flink.ml.preprocessing.PolynomialFeatures.Degree
-
-import scala.reflect.ClassTag
-
-/** Maps a vector into the polynomial feature space.
- *
- * This transformer takes a a vector of values `(x, y, z, ...)` and maps it into the
- * polynomial feature space of degree `d`. That is to say, it calculates the following
- * representation:
- *
- * `(x, y, z, x^2, xy, y^2, yz, z^2, x^3, x^2y, x^2z, xyz, ...)^T`
- *
- * This transformer can be prepended to all [[org.apache.flink.ml.pipeline.Transformer]] and
- * [[org.apache.flink.ml.pipeline.Predictor]] implementations which expect an input of
- * [[LabeledVector]].
- *
- * @example
- * {{{
- * val trainingDS: DataSet[LabeledVector] = ...
- *
- * val polyFeatures = PolynomialFeatures()
- * .setDegree(3)
- *
- * val mlr = MultipleLinearRegression()
- *
- * val pipeline = polyFeatures.chainPredictor(mlr)
- *
- * pipeline.fit(trainingDS)
- * }}}
- *
- * =Parameters=
- *
- * - [[org.apache.flink.ml.preprocessing.PolynomialFeatures.Degree]]: Maximum polynomial degree
- */
-class PolynomialFeatures extends Transformer[PolynomialFeatures] {
-
- def setDegree(degree: Int): PolynomialFeatures = {
- parameters.add(Degree, degree)
- this
- }
-}
-
-object PolynomialFeatures{
-
- // ====================================== Parameters =============================================
-
- case object Degree extends Parameter[Int] {
- override val defaultValue: Option[Int] = Some(1)
- }
-
- // =================================== Factory methods ===========================================
-
- def apply(): PolynomialFeatures = {
- new PolynomialFeatures()
- }
-
- // ====================================== Operations =============================================
-
- /** The [[PolynomialFeatures]] transformer does not need a fitting phase.
- *
- * @tparam T The fitting works with arbitrary input types
- * @return
- */
- implicit def fitNoOp[T] = {
- new FitOperation[PolynomialFeatures, T]{
- override def fit(
- instance: PolynomialFeatures,
- fitParameters: ParameterMap,
- input: DataSet[T])
- : Unit = {}
- }
- }
-
- /** [[org.apache.flink.ml.pipeline.TransformDataSetOperation]] to map a [[Vector]] into the
- * polynomial feature space.
- *
- * @tparam T Subclass of [[Vector]]
- * @return
- */
- implicit def transformVectorIntoPolynomialBase[
- T <: Vector : VectorBuilder: TypeInformation: ClassTag
- ] = {
- new TransformDataSetOperation[PolynomialFeatures, T, T] {
- override def transformDataSet(
- instance: PolynomialFeatures,
- transformParameters: ParameterMap,
- input: DataSet[T])
- : DataSet[T] = {
- val resultingParameters = instance.parameters ++ transformParameters
-
- val degree = resultingParameters(Degree)
-
- input.map {
- vector: T => {
- calculatePolynomial(degree, vector)
- }
- }
- }
- }
- }
-
- /** [[org.apache.flink.ml.pipeline.TransformDataSetOperation]] to map a [[LabeledVector]] into the
- * polynomial feature space
- */
- implicit val transformLabeledVectorIntoPolynomialBase =
- new TransformDataSetOperation[PolynomialFeatures, LabeledVector, LabeledVector] {
-
- override def transformDataSet(
- instance: PolynomialFeatures,
- transformParameters: ParameterMap,
- input: DataSet[LabeledVector])
- : DataSet[LabeledVector] = {
- val resultingParameters = instance.parameters ++ transformParameters
-
- val degree = resultingParameters(Degree)
-
- input.map {
- labeledVector => {
- val vector = labeledVector.vector
- val label = labeledVector.label
-
- val transformedVector = calculatePolynomial(degree, vector)
-
- LabeledVector(label, transformedVector)
- }
- }
- }
- }
-
-
- private def calculatePolynomial[T <: Vector: VectorBuilder](degree: Int, vector: T): T = {
- val builder = implicitly[VectorBuilder[T]]
- builder.build(calculateCombinedCombinations(degree, vector))
- }
-
- /** Calculates for a given vector its representation in the polynomial feature space.
- *
- * @param degree Maximum degree of polynomial
- * @param vector Values of the polynomial variables
- * @return List of polynomial values
- */
- private def calculateCombinedCombinations(degree: Int, vector: Vector): List[Double] = {
- if(degree == 0) {
- List()
- } else {
- val partialResult = calculateCombinedCombinations(degree - 1, vector)
-
- val combinations = calculateCombinations(vector.size, degree)
-
- val result = combinations map {
- combination =>
- combination.zipWithIndex.map{
- case (exp, idx) => math.pow(vector(idx), exp)
- }.fold(1.0)(_ * _)
- }
-
- result ::: partialResult
- }
-
- }
-
- /** Calculates all possible combinations of a polynom of degree `value`, whereas the polynom
- * can consist of up to `length` factors. The return value is the list of the exponents of the
- * individual factors
- *
- * @param length maximum number of factors
- * @param value degree of polynomial
- * @return List of lists which contain the exponents of the individual factors
- */
- private def calculateCombinations(length: Int, value: Int): List[List[Int]] = {
- if(length == 0) {
- List()
- } else if (length == 1) {
- List(List(value))
- } else {
- value to 0 by -1 flatMap {
- v =>
- calculateCombinations(length - 1, value - v) map {
- v::_
- }
- } toList
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala
deleted file mode 100644
index c8bf0e7..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/Splitter.scala
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.preprocessing
-
-import org.apache.flink.api.common.typeinfo.{TypeInformation, BasicTypeInfo}
-import org.apache.flink.api.java.Utils
-import org.apache.flink.api.scala._
-import org.apache.flink.api.scala.DataSet
-import org.apache.flink.api.scala.utils._
-
-
-import org.apache.flink.ml.common.{FlinkMLTools, ParameterMap, WithParameters}
-import org.apache.flink.util.Collector
-import _root_.scala.reflect.ClassTag
-
-object Splitter {
-
- case class TrainTestDataSet[T: TypeInformation : ClassTag](training: DataSet[T],
- testing: DataSet[T])
-
- case class TrainTestHoldoutDataSet[T: TypeInformation : ClassTag](training: DataSet[T],
- testing: DataSet[T],
- holdout: DataSet[T])
- // --------------------------------------------------------------------------------------------
- // randomSplit
- // --------------------------------------------------------------------------------------------
- /**
- * Split a DataSet by the probability fraction of each element.
- *
- * @param input DataSet to be split
- * @param fraction Probability that each element is chosen, should be [0,1] This fraction
- * refers to the first element in the resulting array.
- * @param precise Sampling by default is random and can result in slightly lop-sided
- * sample sets. When precise is true, equal sample set size are forced,
- * however this is somewhat less efficient.
- * @param seed Random number generator seed.
- * @return An array of two datasets
- */
-
- def randomSplit[T: TypeInformation : ClassTag](
- input: DataSet[T],
- fraction: Double,
- precise: Boolean = false,
- seed: Long = Utils.RNG.nextLong())
- : Array[DataSet[T]] = {
- import org.apache.flink.api.scala._
-
- val indexedInput: DataSet[(Long, T)] = input.zipWithUniqueId
-
- if ((fraction >= 1) || (fraction <= 0)) {
- throw new IllegalArgumentException("sampling fraction outside of (0,1) bounds is nonsensical")
- }
-
- val leftSplit: DataSet[(Long, T)] = precise match {
- case false => indexedInput.sample(false, fraction, seed)
- case true => {
- val count = indexedInput.count() // todo: count only needed for precise and kills perf.
- val numOfSamples = math.round(fraction * count).toInt
- indexedInput.sampleWithSize(false, numOfSamples, seed)
- }
- }
-
- val leftSplitLight = leftSplit.map((o: (Long, T)) => (o._1, false))
-
- val rightSplit: DataSet[T] = indexedInput.leftOuterJoin[(Long, Boolean)](leftSplitLight)
- .where(0)
- .equalTo(0).apply {
- (full: (Long,T) , left: (Long, Boolean), collector: Collector[T]) =>
- if (left == null) {
- collector.collect(full._2)
- }
- }
-
- Array(leftSplit.map((o: (Long, T)) => o._2), rightSplit)
- }
-
- // --------------------------------------------------------------------------------------------
- // multiRandomSplit
- // --------------------------------------------------------------------------------------------
- /**
- * Split a DataSet by the probability fraction of each element of a vector.
- *
- * @param input DataSet to be split
- * @param fracArray An array of PROPORTIONS for splitting the DataSet. Unlike the
- * randomSplit function, number greater than 1 do not lead to over
- * sampling. The number of splits is dictated by the length of this array.
- * The number are normalized, eg. Array(1.0, 2.0) would yield
- * two data sets with a 33/66% split.
- * @param seed Random number generator seed.
- * @return An array of DataSets whose length is equal to the length of fracArray
- */
- def multiRandomSplit[T: TypeInformation : ClassTag](
- input: DataSet[T],
- fracArray: Array[Double],
- seed: Long = Utils.RNG.nextLong())
- : Array[DataSet[T]] = {
-
- import org.apache.commons.math3.distribution.EnumeratedIntegerDistribution
-
- val eid = new EnumeratedIntegerDistribution((0 to fracArray.length - 1).toArray, fracArray)
-
- eid.reseedRandomGenerator(seed)
-
- val tempDS: DataSet[(Int,T)] = input.map((o: T) => (eid.sample, o))
-
- val splits = fracArray.length
- val outputArray = new Array[DataSet[T]]( splits )
-
- for (k <- 0 to splits-1){
- outputArray(k) = tempDS.filter((o: (Int, T)) => o._1 == k)
- .map((o: (Int, T)) => o._2)
- }
-
- outputArray
- }
-
- // --------------------------------------------------------------------------------------------
- // kFoldSplit
- // --------------------------------------------------------------------------------------------
- /**
- * Split a DataSet into an array of TrainTest DataSets
- *
- * @param input DataSet to be split
- * @param kFolds The number of TrainTest DataSets to be returns. Each 'testing' will be
- * 1/k of the dataset, randomly sampled, the training will be the remainder
- * of the dataset. The DataSet is split into kFolds first, so that no
- * observation will occuring in multiple folds.
- * @param seed Random number generator seed.
- * @return An array of TrainTestDataSets
- */
- def kFoldSplit[T: TypeInformation : ClassTag](
- input: DataSet[T],
- kFolds: Int,
- seed: Long = Utils.RNG.nextLong())
- : Array[TrainTestDataSet[T]] = {
-
- val fracs = Array.fill(kFolds)(1.0)
- val dataSetArray = multiRandomSplit(input, fracs, seed)
-
- dataSetArray.map( ds => TrainTestDataSet(dataSetArray.filter(_ != ds)
- .reduce(_ union _),
- ds))
-
- }
-
- // --------------------------------------------------------------------------------------------
- // trainTestSplit
- // --------------------------------------------------------------------------------------------
- /**
- * A wrapper for randomSplit that yields a TrainTestDataSet
- *
- * @param input DataSet to be split
- * @param fraction Probability that each element is chosen, should be [0,1].
- * This fraction refers to the training element in TrainTestSplit
- * @param precise Sampling by default is random and can result in slightly lop-sided
- * sample sets. When precise is true, equal sample set size are forced,
- * however this is somewhat less efficient.
- * @param seed Random number generator seed.
- * @return A TrainTestDataSet
- */
- def trainTestSplit[T: TypeInformation : ClassTag](
- input: DataSet[T],
- fraction: Double = 0.6,
- precise: Boolean = false,
- seed: Long = Utils.RNG.nextLong())
- : TrainTestDataSet[T] = {
- val dataSetArray = randomSplit(input, fraction, precise, seed)
- TrainTestDataSet(dataSetArray(0), dataSetArray(1))
- }
-
- // --------------------------------------------------------------------------------------------
- // trainTestHoldoutSplit
- // --------------------------------------------------------------------------------------------
- /**
- * A wrapper for multiRandomSplit that yields a TrainTestHoldoutDataSet
- *
- * @param input DataSet to be split
- * @param fracTuple A tuple of three doubles, where the first element specifies the
- * size of the training set, the second element the testing set, and
- * the third element is the holdout set. These are proportional and
- * will be normalized internally.
- * @param seed Random number generator seed.
- * @return A TrainTestDataSet
- */
- def trainTestHoldoutSplit[T: TypeInformation : ClassTag](
- input: DataSet[T],
- fracTuple: Tuple3[Double, Double, Double] = (0.6,0.3,0.1),
- seed: Long = Utils.RNG.nextLong())
- : TrainTestHoldoutDataSet[T] = {
- val fracArray = Array(fracTuple._1, fracTuple._2, fracTuple._3)
- val dataSetArray = multiRandomSplit(input, fracArray, seed)
- TrainTestHoldoutDataSet(dataSetArray(0), dataSetArray(1), dataSetArray(2))
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
deleted file mode 100644
index aa38f41..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
+++ /dev/null
@@ -1,302 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.preprocessing
-
-import breeze.linalg
-import breeze.numerics.sqrt
-import breeze.numerics.sqrt._
-import org.apache.flink.api.common.typeinfo.TypeInformation
-import org.apache.flink.api.scala._
-import org.apache.flink.ml.common.{LabeledVector, Parameter, ParameterMap}
-import org.apache.flink.ml.math.Breeze._
-import org.apache.flink.ml.math.{BreezeVectorConverter, Vector}
-import org.apache.flink.ml.pipeline.{TransformOperation, FitOperation,
-Transformer}
-import org.apache.flink.ml.preprocessing.StandardScaler.{Mean, Std}
-
-import scala.reflect.ClassTag
-
-/** Scales observations, so that all features have a user-specified mean and standard deviation.
- * By default for [[StandardScaler]] transformer mean=0.0 and std=1.0.
- *
- * This transformer takes a subtype of [[Vector]] of values and maps it to a
- * scaled subtype of [[Vector]] such that each feature has a user-specified mean and standard
- * deviation.
- *
- * This transformer can be prepended to all [[Transformer]] and
- * [[org.apache.flink.ml.pipeline.Predictor]] implementations which expect as input a subtype
- * of [[Vector]].
- *
- * @example
- * {{{
- * val trainingDS: DataSet[Vector] = env.fromCollection(data)
- * val transformer = StandardScaler().setMean(10.0).setStd(2.0)
- *
- * transformer.fit(trainingDS)
- * val transformedDS = transformer.transform(trainingDS)
- * }}}
- *
- * =Parameters=
- *
- * - [[Mean]]: The mean value of transformed data set; by default equal to 0
- * - [[Std]]: The standard deviation of the transformed data set; by default
- * equal to 1
- */
-class StandardScaler extends Transformer[StandardScaler] {
-
- private[preprocessing] var metricsOption: Option[
- DataSet[(linalg.Vector[Double], linalg.Vector[Double])]
- ] = None
-
- /** Sets the target mean of the transformed data
- *
- * @param mu the user-specified mean value.
- * @return the StandardScaler instance with its mean value set to the user-specified value
- */
- def setMean(mu: Double): StandardScaler = {
- parameters.add(Mean, mu)
- this
- }
-
- /** Sets the target standard deviation of the transformed data
- *
- * @param std the user-specified std value. In case the user gives 0.0 value as input,
- * the std is set to the default value: 1.0.
- * @return the StandardScaler instance with its std value set to the user-specified value
- */
- def setStd(std: Double): StandardScaler = {
- if (std == 0.0) {
- return this
- }
- parameters.add(Std, std)
- this
- }
-}
-
-object StandardScaler {
-
- // ====================================== Parameters =============================================
-
- case object Mean extends Parameter[Double] {
- override val defaultValue: Option[Double] = Some(0.0)
- }
-
- case object Std extends Parameter[Double] {
- override val defaultValue: Option[Double] = Some(1.0)
- }
-
- // ==================================== Factory methods ==========================================
-
- def apply(): StandardScaler = {
- new StandardScaler()
- }
-
- // ====================================== Operations =============================================
-
- /** Trains the [[org.apache.flink.ml.preprocessing.StandardScaler]] by learning the mean and
- * standard deviation of the training data. These values are used inthe transform step
- * to transform the given input data.
- *
- * @tparam T Input data type which is a subtype of [[Vector]]
- * @return
- */
- implicit def fitVectorStandardScaler[T <: Vector] = new FitOperation[StandardScaler, T] {
- override def fit(instance: StandardScaler, fitParameters: ParameterMap, input: DataSet[T])
- : Unit = {
- val metrics = extractFeatureMetrics(input)
-
- instance.metricsOption = Some(metrics)
- }
- }
-
- /** Trains the [[StandardScaler]] by learning the mean and standard deviation of the training
- * data which is of type [[LabeledVector]]. The mean and standard deviation are used to
- * transform the given input data.
- *
- */
- implicit val fitLabeledVectorStandardScaler = {
- new FitOperation[StandardScaler, LabeledVector] {
- override def fit(
- instance: StandardScaler,
- fitParameters: ParameterMap,
- input: DataSet[LabeledVector])
- : Unit = {
- val vectorDS = input.map(_.vector)
- val metrics = extractFeatureMetrics(vectorDS)
-
- instance.metricsOption = Some(metrics)
- }
- }
- }
-
- /** Trains the [[StandardScaler]] by learning the mean and standard deviation of the training
- * data which is of type ([[Vector]], Double). The mean and standard deviation are used to
- * transform the given input data.
- *
- */
- implicit def fitLabelVectorTupleStandardScaler
- [T <: Vector: BreezeVectorConverter: TypeInformation: ClassTag] = {
- new FitOperation[StandardScaler, (T, Double)] {
- override def fit(
- instance: StandardScaler,
- fitParameters: ParameterMap,
- input: DataSet[(T, Double)])
- : Unit = {
- val vectorDS = input.map( (i: (T, Double)) => i._1)
- val metrics = extractFeatureMetrics(vectorDS)
-
- instance.metricsOption = Some(metrics)
- }
- }
- }
-
- /** Calculates in one pass over the data the features' mean and standard deviation.
- * For the calculation of the Standard deviation with one pass over the data,
- * the Youngs & Cramer algorithm was used:
- * [[http://www.cs.yale.edu/publications/techreports/tr222.pdf]]
- *
- *
- * @param dataSet The data set for which we want to calculate mean and variance
- * @return DataSet containing a single tuple of two vectors (meanVector, stdVector).
- * The first vector represents the mean vector and the second is the standard
- * deviation vector.
- */
- private def extractFeatureMetrics[T <: Vector](dataSet: DataSet[T])
- : DataSet[(linalg.Vector[Double], linalg.Vector[Double])] = {
- val metrics = dataSet.map{
- v: T => (1.0, v.asBreeze, linalg.Vector.zeros[Double](v.size))
- }.reduce{
- (metrics1, metrics2) => {
- /* We use formula 1.5b of the cited technical report for the combination of partial
- * sum of squares. According to 1.5b:
- * val temp1 : m/n(m+n)
- * val temp2 : n/m
- */
- val temp1 = metrics1._1 / (metrics2._1 * (metrics1._1 + metrics2._1))
- val temp2 = metrics2._1 / metrics1._1
- val tempVector = (metrics1._2 * temp2) - metrics2._2
- val tempS = (metrics1._3 + metrics2._3) + (tempVector :* tempVector) * temp1
-
- (metrics1._1 + metrics2._1, metrics1._2 + metrics2._2, tempS)
- }
- }.map{
- metric => {
- val varianceVector = sqrt(metric._3 / metric._1)
-
- for (i <- 0 until varianceVector.size) {
- if (varianceVector(i) == 0.0) {
- varianceVector.update(i, 1.0)
- }
- }
- (metric._2 / metric._1, varianceVector)
- }
- }
- metrics
- }
-
- /** Base class for StandardScaler's [[TransformOperation]]. This class has to be extended for
- * all types which are supported by [[StandardScaler]]'s transform operation.
- *
- * @tparam T
- */
- abstract class StandardScalerTransformOperation[T: TypeInformation: ClassTag]
- extends TransformOperation[
- StandardScaler,
- (linalg.Vector[Double], linalg.Vector[Double]),
- T,
- T] {
-
- var mean: Double = _
- var std: Double = _
-
- override def getModel(
- instance: StandardScaler,
- transformParameters: ParameterMap)
- : DataSet[(linalg.Vector[Double], linalg.Vector[Double])] = {
- mean = transformParameters(Mean)
- std = transformParameters(Std)
-
- instance.metricsOption match {
- case Some(metrics) => metrics
- case None =>
- throw new RuntimeException("The StandardScaler has not been fitted to the data. " +
- "This is necessary to estimate the mean and standard deviation of the data.")
- }
- }
-
- def scale[V <: Vector: BreezeVectorConverter](
- vector: V,
- model: (linalg.Vector[Double], linalg.Vector[Double]))
- : V = {
- val (broadcastMean, broadcastStd) = model
- var myVector = vector.asBreeze
- myVector -= broadcastMean
- myVector :/= broadcastStd
- myVector = (myVector :* std) + mean
- myVector.fromBreeze
- }
- }
-
- /** [[TransformOperation]] to transform [[Vector]] types
- *
- * @tparam T
- * @return
- */
- implicit def transformVectors[T <: Vector: BreezeVectorConverter: TypeInformation: ClassTag] = {
- new StandardScalerTransformOperation[T]() {
- override def transform(
- vector: T,
- model: (linalg.Vector[Double], linalg.Vector[Double]))
- : T = {
- scale(vector, model)
- }
- }
- }
-
- /** [[TransformOperation]] to transform tuples of type ([[Vector]], [[Double]]).
- *
- * @tparam T
- * @return
- */
- implicit def transformTupleVectorDouble[
- T <: Vector: BreezeVectorConverter: TypeInformation: ClassTag] = {
- new StandardScalerTransformOperation[(T, Double)] {
- override def transform(
- element: (T, Double),
- model: (linalg.Vector[Double], linalg.Vector[Double]))
- : (T, Double) = {
- (scale(element._1, model), element._2)
- }
- }
- }
-
- /** [[TransformOperation]] to transform [[LabeledVector]].
- *
- */
- implicit val transformLabeledVector = new StandardScalerTransformOperation[LabeledVector] {
- override def transform(
- element: LabeledVector,
- model: (linalg.Vector[Double], linalg.Vector[Double]))
- : LabeledVector = {
- val LabeledVector(label, vector) = element
-
- LabeledVector(label, scale(vector, model))
- }
- }
-}
diff --git a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala b/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala
deleted file mode 100644
index 2e2e35a..0000000
--- a/flink-libraries/flink-ml/src/main/scala/org/apache/flink/ml/recommendation/ALS.scala
+++ /dev/null
@@ -1,1060 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.ml.recommendation
-
-import java.{util, lang}
-
-import org.apache.flink.api.common.operators.base.JoinOperatorBase.JoinHint
-import org.apache.flink.api.scala._
-import org.apache.flink.api.common.operators.Order
-import org.apache.flink.core.memory.{DataOutputView, DataInputView}
-import org.apache.flink.ml.common._
-import org.apache.flink.ml.pipeline.{FitOperation, PredictDataSetOperation, Predictor}
-import org.apache.flink.types.Value
-import org.apache.flink.util.Collector
-import org.apache.flink.api.common.functions.{Partitioner => FlinkPartitioner, GroupReduceFunction, CoGroupFunction}
-
-import com.github.fommil.netlib.BLAS.{ getInstance => blas }
-import com.github.fommil.netlib.LAPACK.{ getInstance => lapack }
-import org.netlib.util.intW
-
-import scala.collection.mutable
-import scala.collection.mutable.ArrayBuffer
-import scala.util.Random
-
-/** Alternating least squares algorithm to calculate a matrix factorization.
- *
- * Given a matrix `R`, ALS calculates two matrices `U` and `V` such that `R ~~ U^TV`. The
- * unknown row dimension is given by the number of latent factors. Since matrix factorization
- * is often used in the context of recommendation, we'll call the first matrix the user and the
- * second matrix the item matrix. The `i`th column of the user matrix is `u_i` and the `i`th
- * column of the item matrix is `v_i`. The matrix `R` is called the ratings matrix and
- * `(R)_{i,j} = r_{i,j}`.
- *
- * In order to find the user and item matrix, the following problem is solved:
- *
- * `argmin_{U,V} sum_(i,j\ with\ r_{i,j} != 0) (r_{i,j} - u_{i}^Tv_{j})^2 +
- * lambda (sum_(i) n_{u_i} ||u_i||^2 + sum_(j) n_{v_j} ||v_j||^2)`
- *
- * with `\lambda` being the regularization factor, `n_{u_i}` being the number of items the user `i`
- * has rated and `n_{v_j}` being the number of times the item `j` has been rated. This
- * regularization scheme to avoid overfitting is called weighted-lambda-regularization. Details
- * can be found in the work of [[http://dx.doi.org/10.1007/978-3-540-68880-8_32 Zhou et al.]].
- *
- * By fixing one of the matrices `U` or `V` one obtains a quadratic form which can be solved. The
- * solution of the modified problem is guaranteed to decrease the overall cost function. By
- * applying this step alternately to the matrices `U` and `V`, we can iteratively improve the
- * matrix factorization.
- *
- * The matrix `R` is given in its sparse representation as a tuple of `(i, j, r)` where `i` is the
- * row index, `j` is the column index and `r` is the matrix value at position `(i,j)`.
- *
- * @example
- * {{{
- * val inputDS: DataSet[(Int, Int, Double)] = env.readCsvFile[(Int, Int, Double)](
- * pathToTrainingFile)
- *
- * val als = ALS()
- * .setIterations(10)
- * .setNumFactors(10)
- *
- * als.fit(inputDS)
- *
- * val data2Predict: DataSet[(Int, Int)] = env.readCsvFile[(Int, Int)](pathToData)
- *
- * als.predict(data2Predict)
- * }}}
- *
- * =Parameters=
- *
- * - [[org.apache.flink.ml.recommendation.ALS.NumFactors]]:
- * The number of latent factors. It is the dimension of the calculated user and item vectors.
- * (Default value: '''10''')
- *
- * - [[org.apache.flink.ml.recommendation.ALS.Lambda]]:
- * Regularization factor. Tune this value in order to avoid overfitting/generalization.
- * (Default value: '''1''')
- *
- * - [[org.apache.flink.ml.regression.MultipleLinearRegression.Iterations]]:
- * The number of iterations to perform. (Default value: '''10''')
- *
- * - [[org.apache.flink.ml.recommendation.ALS.Blocks]]:
- * The number of blocks into which the user and item matrix a grouped. The fewer
- * blocks one uses, the less data is sent redundantly. However, bigger blocks entail bigger
- * update messages which have to be stored on the Heap. If the algorithm fails because of
- * an OutOfMemoryException, then try to increase the number of blocks. (Default value: '''None''')
- *
- * - [[org.apache.flink.ml.recommendation.ALS.Seed]]:
- * Random seed used to generate the initial item matrix for the algorithm.
- * (Default value: '''0''')
- *
- * - [[org.apache.flink.ml.recommendation.ALS.TemporaryPath]]:
- * Path to a temporary directory into which intermediate results are stored. If
- * this value is set, then the algorithm is split into two preprocessing steps, the ALS iteration
- * and a post-processing step which calculates a last ALS half-step. The preprocessing steps
- * calculate the [[org.apache.flink.ml.recommendation.ALS.OutBlockInformation]] and
- * [[org.apache.flink.ml.recommendation.ALS.InBlockInformation]] for the given rating matrix.
- * The result of the individual steps are stored in the specified directory. By splitting the
- * algorithm into multiple smaller steps, Flink does not have to split the available memory
- * amongst too many operators. This allows the system to process bigger individual messasges and
- * improves the overall performance. (Default value: '''None''')
- *
- * The ALS implementation is based on Spark's MLLib implementation of ALS which you can find
- * [[https://github.com/apache/spark/blob/master/mllib/src/main/scala/org/apache/spark/mllib/
- * recommendation/ALS.scala here]].
- */
-class ALS extends Predictor[ALS] {
-
- import ALS._
-
- // Stores the matrix factorization after the fitting phase
- var factorsOption: Option[(DataSet[Factors], DataSet[Factors])] = None
-
- /** Sets the number of latent factors/row dimension of the latent model
- *
- * @param numFactors
- * @return
- */
- def setNumFactors(numFactors: Int): ALS = {
- parameters.add(NumFactors, numFactors)
- this
- }
-
- /** Sets the regularization coefficient lambda
- *
- * @param lambda
- * @return
- */
- def setLambda(lambda: Double): ALS = {
- parameters.add(Lambda, lambda)
- this
- }
-
- /** Sets the number of iterations of the ALS algorithm
- *
- * @param iterations
- * @return
- */
- def setIterations(iterations: Int): ALS = {
- parameters.add(Iterations, iterations)
- this
- }
-
- /** Sets the number of blocks into which the user and item matrix shall be partitioned
- *
- * @param blocks
- * @return
- */
- def setBlocks(blocks: Int): ALS = {
- parameters.add(Blocks, blocks)
- this
- }
-
- /** Sets the random seed for the initial item matrix initialization
- *
- * @param seed
- * @return
- */
- def setSeed(seed: Long): ALS = {
- parameters.add(Seed, seed)
- this
- }
-
- /** Sets the temporary path into which intermediate results are written in order to increase
- * performance.
- *
- * @param temporaryPath
- * @return
- */
- def setTemporaryPath(temporaryPath: String): ALS = {
- parameters.add(TemporaryPath, temporaryPath)
- this
- }
-
- /** Empirical risk of the trained model (matrix factorization).
- *
- * @param labeledData Reference data
- * @param riskParameters Additional parameters for the empirical risk calculation
- * @return
- */
- def empiricalRisk(
- labeledData: DataSet[(Long, Long, Double)],
- riskParameters: ParameterMap = ParameterMap.Empty)
- : DataSet[Double] = {
- val resultingParameters = parameters ++ riskParameters
-
- val lambda = resultingParameters(Lambda)
-
- val data = labeledData map {
- x => (x._1, x._2)
- }
-
- factorsOption match {
- case Some((userFactors, itemFactors)) => {
- val predictions = data.join(userFactors, JoinHint.REPARTITION_HASH_SECOND).where(0)
- .equalTo(0).join(itemFactors, JoinHint.REPARTITION_HASH_SECOND).where("_1._2")
- .equalTo(0).map {
- triple => {
- val (((uID, iID), uFactors), iFactors) = triple
-
- val uFactorsVector = uFactors.factors
- val iFactorsVector = iFactors.factors
-
- val squaredUNorm2 = blas.ddot(
- uFactorsVector.length,
- uFactorsVector,
- 1,
- uFactorsVector,
- 1)
- val squaredINorm2 = blas.ddot(
- iFactorsVector.length,
- iFactorsVector,
- 1,
- iFactorsVector,
- 1)
-
- val prediction = blas.ddot(uFactorsVector.length, uFactorsVector, 1, iFactorsVector, 1)
-
- (uID, iID, prediction, squaredUNorm2, squaredINorm2)
- }
- }
-
- labeledData.join(predictions).where(0,1).equalTo(0,1) {
- (left, right) => {
- val (_, _, expected) = left
- val (_, _, predicted, squaredUNorm2, squaredINorm2) = right
-
- val residual = expected - predicted
-
- residual * residual + lambda * (squaredUNorm2 + squaredINorm2)
- }
- } reduce {
- _ + _
- }
- }
-
- case None => throw new RuntimeException("The ALS model has not been fitted to data. " +
- "Prior to predicting values, it has to be trained on data.")
- }
- }
-}
-
-object ALS {
- val USER_FACTORS_FILE = "userFactorsFile"
- val ITEM_FACTORS_FILE = "itemFactorsFile"
-
- // ========================================= Parameters ==========================================
-
- case object NumFactors extends Parameter[Int] {
- val defaultValue: Option[Int] = Some(10)
- }
-
- case object Lambda extends Parameter[Double] {
- val defaultValue: Option[Double] = Some(1.0)
- }
-
- case object Iterations extends Parameter[Int] {
- val defaultValue: Option[Int] = Some(10)
- }
-
- case object Blocks extends Parameter[Int] {
- val defaultValue: Option[Int] = None
- }
-
- case object Seed extends Parameter[Long] {
- val defaultValue: Option[Long] = Some(0L)
- }
-
- case object TemporaryPath extends Parameter[String] {
- val defaultValue: Option[String] = None
- }
-
- // ==================================== ALS type definitions =====================================
-
- /** Representation of a user-item rating
- *
- * @param user User ID of the rating user
- * @param item Item iD of the rated item
- * @param rating Rating value
- */
- case class Rating(user: Long, item: Long, rating: Double)
-
- /** Latent factor model vector
- *
- * @param id
- * @param factors
- */
- case class Factors(id: Long, factors: Array[Double]) {
- override def toString = s"($id, ${factors.mkString(",")})"
- }
-
- case class Factorization(userFactors: DataSet[Factors], itemFactors: DataSet[Factors])
-
- case class OutBlockInformation(elementIDs: Array[Long], outLinks: OutLinks) {
- override def toString: String = {
- s"OutBlockInformation:((${elementIDs.mkString(",")}), ($outLinks))"
- }
- }
-
- class OutLinks(var links: Array[scala.collection.mutable.BitSet]) extends Value {
- def this() = this(null)
-
- override def toString: String = {
- s"${links.mkString("\n")}"
- }
-
- override def write(out: DataOutputView): Unit = {
- out.writeInt(links.length)
- links foreach {
- link => {
- val bitMask = link.toBitMask
- out.writeInt(bitMask.length)
- for (element <- bitMask) {
- out.writeLong(element)
- }
- }
- }
- }
-
- override def read(in: DataInputView): Unit = {
- val length = in.readInt()
- links = new Array[scala.collection.mutable.BitSet](length)
-
- for (i <- 0 until length) {
- val bitMaskLength = in.readInt()
- val bitMask = new Array[Long](bitMaskLength)
- for (j <- 0 until bitMaskLength) {
- bitMask(j) = in.readLong()
- }
- links(i) = mutable.BitSet.fromBitMask(bitMask)
- }
- }
-
- def apply(idx: Int) = links(idx)
- }
-
- case class InBlockInformation(elementIDs: Array[Long], ratingsForBlock: Array[BlockRating]) {
-
- override def toString: String = {
- s"InBlockInformation:((${elementIDs.mkString(",")}), (${ratingsForBlock.mkString("\n")}))"
- }
- }
-
- case class BlockRating(var ratings: Array[(Array[Int], Array[Double])]) {
- def apply(idx: Int) = ratings(idx)
-
- override def toString: String = {
- ratings.map {
- case (left, right) => s"((${left.mkString(",")}),(${right.mkString(",")}))"
- }.mkString(",")
- }
- }
-
- case class BlockedFactorization(userFactors: DataSet[(Int, Array[Array[Double]])],
- itemFactors: DataSet[(Int, Array[Array[Double]])])
-
- class BlockIDPartitioner extends FlinkPartitioner[Int] {
- override def partition(blockID: Int, numberOfPartitions: Int): Int = {
- blockID % numberOfPartitions
- }
- }
-
- class BlockIDGenerator(blocks: Int) extends Serializable {
- def apply(id: Long): Int = {
- (id % blocks).toInt
- }
- }
-
- // ================================= Factory methods =============================================
-
- def apply(): ALS = {
- new ALS()
- }
-
- // ===================================== Operations ==============================================
-
- /** Predict operation which calculates the matrix entry for the given indices */
- implicit val predictRating = new PredictDataSetOperation[
- ALS,
- (Long, Long),
- (Long, Long, Double)] {
- override def predictDataSet(
- instance: ALS,
- predictParameters: ParameterMap,
- input: DataSet[(Long, Long)])
- : DataSet[(Long, Long, Double)] = {
-
- instance.factorsOption match {
- case Some((userFactors, itemFactors)) => {
- input.join(userFactors, JoinHint.REPARTITION_HASH_SECOND).where(0).equalTo(0)
- .join(itemFactors, JoinHint.REPARTITION_HASH_SECOND).where("_1._2").equalTo(0).map {
- triple => {
- val (((uID, iID), uFactors), iFactors) = triple
-
- val uFactorsVector = uFactors.factors
- val iFactorsVector = iFactors.factors
-
- val prediction = blas.ddot(
- uFactorsVector.length,
- uFactorsVector,
- 1,
- iFactorsVector,
- 1)
-
- (uID, iID, prediction)
- }
- }
- }
-
- case None => throw new RuntimeException("The ALS model has not been fitted to data. " +
- "Prior to predicting values, it has to be trained on data.")
- }
- }
- }
-
- implicit val predictRatingInt = new PredictDataSetOperation[ALS, (Int, Int), (Int, Int, Double)] {
- override def predictDataSet(
- instance: ALS,
- predictParameters: ParameterMap,
- input: DataSet[(Int, Int)])
- : DataSet[(Int, Int, Double)] = {
- val longInput = input.map { x => (x._1.toLong, x._2.toLong)}
-
- val longResult = implicitly[PredictDataSetOperation[ALS, (Long, Long), (Long, Long, Double)]]
- .predictDataSet(
- instance,
- predictParameters,
- longInput)
-
- longResult.map{ x => (x._1.toInt, x._2.toInt, x._3)}
- }
- }
-
- /** Calculates the matrix factorization for the given ratings. A rating is defined as
- * a tuple of user ID, item ID and the corresponding rating.
- *
- * @return Factorization containing the user and item matrix
- */
- implicit val fitALS = new FitOperation[ALS, (Long, Long, Double)] {
- override def fit(
- instance: ALS,
- fitParameters: ParameterMap,
- input: DataSet[(Long, Long, Double)])
- : Unit = {
- val resultParameters = instance.parameters ++ fitParameters
-
- val userBlocks = resultParameters.get(Blocks).getOrElse(input.count.toInt)
- val itemBlocks = userBlocks
- val persistencePath = resultParameters.get(TemporaryPath)
- val seed = resultParameters(Seed)
- val factors = resultParameters(NumFactors)
- val iterations = resultParameters(Iterations)
- val lambda = resultParameters(Lambda)
-
- val ratings = input.map {
- entry => {
- val (userID, itemID, rating) = entry
- Rating(userID, itemID, rating)
- }
- }
-
- val blockIDPartitioner = new BlockIDPartitioner()
-
- val ratingsByUserBlock = ratings.map{
- rating =>
- val blockID = (rating.user % userBlocks).toInt
- (blockID, rating)
- } partitionCustom(blockIDPartitioner, 0)
-
- val ratingsByItemBlock = ratings map {
- rating =>
- val blockID = (rating.item % itemBlocks).toInt
- (blockID, new Rating(rating.item, rating.user, rating.rating))
- } partitionCustom(blockIDPartitioner, 0)
-
- val (uIn, uOut) = createBlockInformation(userBlocks, itemBlocks, ratingsByUserBlock,
- blockIDPartitioner)
- val (iIn, iOut) = createBlockInformation(itemBlocks, userBlocks, ratingsByItemBlock,
- blockIDPartitioner)
-
- val (userIn, userOut) = persistencePath match {
- case Some(path) => FlinkMLTools.persist(uIn, uOut, path + "userIn", path + "userOut")
- case None => (uIn, uOut)
- }
-
- val (itemIn, itemOut) = persistencePath match {
- case Some(path) => FlinkMLTools.persist(iIn, iOut, path + "itemIn", path + "itemOut")
- case None => (iIn, iOut)
- }
-
- val initialItems = itemOut.partitionCustom(blockIDPartitioner, 0).map{
- outInfos =>
- val blockID = outInfos._1
- val infos = outInfos._2
-
- (blockID, infos.elementIDs.map{
- id =>
- val random = new Random(id ^ seed)
- randomFactors(factors, random)
- })
- }.withForwardedFields("0")
-
- // iteration to calculate the item matrix
- val items = initialItems.iterate(iterations) {
- items => {
- val users = updateFactors(userBlocks, items, itemOut, userIn, factors, lambda,
- blockIDPartitioner)
- updateFactors(itemBlocks, users, userOut, itemIn, factors, lambda, blockIDPartitioner)
- }
- }
-
- val pItems = persistencePath match {
- case Some(path) => FlinkMLTools.persist(items, path + "items")
- case None => items
- }
-
- // perform last half-step to calculate the user matrix
- val users = updateFactors(userBlocks, pItems, itemOut, userIn, factors, lambda,
- blockIDPartitioner)
-
- instance.factorsOption = Some((
- unblock(users, userOut, blockIDPartitioner),
- unblock(pItems, itemOut, blockIDPartitioner)))
- }
- }
-
- implicit val fitALSInt = new FitOperation[ALS, (Int, Int, Double)] {
- override def fit(
- instance: ALS,
- fitParameters: ParameterMap,
- input: DataSet[(Int, Int, Double)])
- : Unit = {
-
- val longInput = input.map { x => (x._1.toLong, x._2.toLong, x._3)}
-
- implicitly[FitOperation[ALS, (Long, Long, Double)]].fit(instance, fitParameters, longInput)
- }
- }
-
- /** Calculates a single half step of the ALS optimization. The result is the new value for
- * either the user or item matrix, depending with which matrix the method was called.
- *
- * @param numUserBlocks Number of blocks in the respective dimension
- * @param items Fixed matrix value for the half step
- * @param itemOut Out information to know where to send the vectors
- * @param userIn In information for the cogroup step
- * @param factors Number of latent factors
- * @param lambda Regularization constant
- * @param blockIDPartitioner Custom Flink partitioner
- * @return New value for the optimized matrix (either user or item)
- */
- def updateFactors(numUserBlocks: Int,
- items: DataSet[(Int, Array[Array[Double]])],
- itemOut: DataSet[(Int, OutBlockInformation)],
- userIn: DataSet[(Int, InBlockInformation)],
- factors: Int,
- lambda: Double, blockIDPartitioner: FlinkPartitioner[Int]):
- DataSet[(Int, Array[Array[Double]])] = {
- // send the item vectors to the blocks whose users have rated the items
- val partialBlockMsgs = itemOut.join(items).where(0).equalTo(0).
- withPartitioner(blockIDPartitioner).apply {
- (left, right, col: Collector[(Int, Int, Array[Array[Double]])]) => {
- val blockID = left._1
- val outInfo = left._2
- val factors = right._2
- var userBlock = 0
- var itemIdx = 0
-
- while(userBlock < numUserBlocks){
- itemIdx = 0
- val buffer = new ArrayBuffer[Array[Double]]
- while(itemIdx < outInfo.elementIDs.length){
- if(outInfo.outLinks(userBlock)(itemIdx)){
- buffer += factors(itemIdx)
- }
- itemIdx += 1
- }
-
- if(buffer.nonEmpty){
- // send update message to userBlock
- col.collect(userBlock, blockID, buffer.toArray)
- }
-
- userBlock += 1
- }
- }
- }
-
- // collect the partial update messages and calculate for each user block the new user vectors
- partialBlockMsgs.coGroup(userIn).where(0).equalTo(0).sortFirstGroup(1, Order.ASCENDING).
- withPartitioner(blockIDPartitioner).apply{
- new CoGroupFunction[(Int, Int, Array[Array[Double]]), (Int,
- InBlockInformation), (Int, Array[Array[Double]])](){
-
- // in order to save space, store only the upper triangle of the XtX matrix
- val triangleSize = (factors*factors - factors)/2 + factors
- val matrix = Array.fill(triangleSize)(0.0)
- val fullMatrix = Array.fill(factors * factors)(0.0)
- val userXtX = new ArrayBuffer[Array[Double]]()
- val userXy = new ArrayBuffer[Array[Double]]()
- val numRatings = new ArrayBuffer[Int]()
-
- override def coGroup(left: lang.Iterable[(Int, Int, Array[Array[Double]])],
- right: lang.Iterable[(Int, InBlockInformation)],
- collector: Collector[(Int, Array[Array[Double]])]): Unit = {
- // there is only one InBlockInformation per user block
- val inInfo = right.iterator().next()._2
- val updates = left.iterator()
-
- val numUsers = inInfo.elementIDs.length
- var blockID = -1
-
- var i = 0
-
- // clear old matrices and allocate new ones
- val matricesToClear = if (numUsers > userXtX.length) {
- val oldLength = userXtX.length
-
- while(i < (numUsers - oldLength)) {
- userXtX += Array.fill(triangleSize)(0.0)
- userXy += Array.fill(factors)(0.0)
- numRatings.+=(0)
-
- i += 1
- }
-
- oldLength
- } else {
- numUsers
- }
-
- i = 0
- while(i < matricesToClear){
- numRatings(i) = 0
-
- util.Arrays.fill(userXtX(i), 0.0)
- util.Arrays.fill(userXy(i), 0.0)
-
- i += 1
- }
-
- var itemBlock = 0
-
- // build XtX matrices and Xy vector
- while(updates.hasNext){
- val update = updates.next()
- val blockFactors = update._3
- blockID = update._1
-
- var p = 0
- while(p < blockFactors.length){
- val vector = blockFactors(p)
-
- outerProduct(vector, matrix, factors)
-
- val (users, ratings) = inInfo.ratingsForBlock(itemBlock)(p)
-
- var i = 0
- while (i < users.length) {
- numRatings(users(i)) += 1
- blas.daxpy(matrix.length, 1, matrix, 1, userXtX(users(i)), 1)
- blas.daxpy(vector.length, ratings(i), vector, 1, userXy(users(i)), 1)
-
- i += 1
- }
- p += 1
- }
-
- itemBlock += 1
- }
-
- val array = new Array[Array[Double]](numUsers)
-
- i = 0
- while(i < numUsers){
- generateFullMatrix(userXtX(i), fullMatrix, factors)
-
- var f = 0
-
- // add regularization constant
- while(f < factors){
- fullMatrix(f*factors + f) += lambda * numRatings(i)
- f += 1
- }
-
- // calculate new user vector
- val result = new intW(0)
- lapack.dposv("U", factors, 1, fullMatrix, factors , userXy(i), factors, result)
- array(i) = userXy(i)
-
- i += 1
- }
-
- collector.collect((blockID, array))
- }
- }
- }.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0")
- }
-
- /** Creates the meta information needed to route the item and user vectors to the respective user
- * and item blocks.
- * * @param userBlocks
- * @param itemBlocks
- * @param ratings
- * @param blockIDPartitioner
- * @return
- */
- def createBlockInformation(userBlocks: Int, itemBlocks: Int, ratings: DataSet[(Int, Rating)],
- blockIDPartitioner: BlockIDPartitioner):
- (DataSet[(Int, InBlockInformation)], DataSet[(Int, OutBlockInformation)]) = {
- val blockIDGenerator = new BlockIDGenerator(itemBlocks)
-
- val usersPerBlock = createUsersPerBlock(ratings)
-
- val outBlockInfos = createOutBlockInformation(ratings, usersPerBlock, itemBlocks,
- blockIDGenerator)
-
- val inBlockInfos = createInBlockInformation(ratings, usersPerBlock, blockIDGenerator)
-
- (inBlockInfos, outBlockInfos)
- }
-
- /** Calculates the userIDs in ascending order of each user block
- *
- * @param ratings
- * @return
- */
- def createUsersPerBlock(ratings: DataSet[(Int, Rating)]): DataSet[(Int, Array[Long])] = {
- ratings.map{ x => (x._1, x._2.user)}.withForwardedFields("0").groupBy(0).
- sortGroup(1, Order.ASCENDING).reduceGroup {
- users => {
- val result = ArrayBuffer[Long]()
- var id = -1
- var oldUser = -1L
-
- while(users.hasNext) {
- val user = users.next()
-
- id = user._1
-
- if (user._2 != oldUser) {
- result.+=(user._2)
- oldUser = user._2
- }
- }
-
- val userIDs = result.toArray
- (id, userIDs)
- }
- }.withForwardedFields("0")
- }
-
- /** Creates the outgoing block information
- *
- * Creates for every user block the outgoing block information. The out block information
- * contains for every item block a [[scala.collection.mutable.BitSet]] which indicates which
- * user vector has to be sent to this block. If a vector v has to be sent to a block b, then
- * bitsets(b)'s bit v is set to 1, otherwise 0. Additionally the user IDataSet are replaced by
- * the user vector's index value.
- *
- * @param ratings
- * @param usersPerBlock
- * @param itemBlocks
- * @param blockIDGenerator
- * @return
- */
- def createOutBlockInformation(ratings: DataSet[(Int, Rating)],
- usersPerBlock: DataSet[(Int, Array[Long])],
- itemBlocks: Int, blockIDGenerator: BlockIDGenerator):
- DataSet[(Int, OutBlockInformation)] = {
- ratings.coGroup(usersPerBlock).where(0).equalTo(0).apply {
- (ratings, users) =>
- val userIDs = users.next()._2
- val numUsers = userIDs.length
-
- val userIDToPos = userIDs.zipWithIndex.toMap
-
- val shouldDataSend = Array.fill(itemBlocks)(new scala.collection.mutable.BitSet(numUsers))
- var blockID = -1
- while (ratings.hasNext) {
- val r = ratings.next()
-
- val pos =
- try {
- userIDToPos(r._2.user)
- }catch{
- case e: NoSuchElementException =>
- throw new RuntimeException(s"Key ${r._2.user} not found. BlockID $blockID. " +
- s"Elements in block ${userIDs.take(5).mkString(", ")}. " +
- s"UserIDList contains ${userIDs.contains(r._2.user)}.", e)
- }
-
- blockID = r._1
- shouldDataSend(blockIDGenerator(r._2.item))(pos) = true
- }
-
- (blockID, OutBlockInformation(userIDs, new OutLinks(shouldDataSend)))
- }.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0")
- }
-
- /** Creates the incoming block information
- *
- * Creates for every user block the incoming block information. The incoming block information
- * contains the userIDs of the users in the respective block and for every item block a
- * BlockRating instance. The BlockRating instance describes for every incoming set of item
- * vectors of an item block, which user rated these items and what the rating was. For that
- * purpose it contains for every incoming item vector a tuple of an id array us and a rating
- * array rs. The array us contains the indices of the users having rated the respective
- * item vector with the ratings in rs.
- *
- * @param ratings
- * @param usersPerBlock
- * @param blockIDGenerator
- * @return
- */
- def createInBlockInformation(ratings: DataSet[(Int, Rating)],
- usersPerBlock: DataSet[(Int, Array[Long])],
- blockIDGenerator: BlockIDGenerator):
- DataSet[(Int, InBlockInformation)] = {
- // Group for every user block the users which have rated the same item and collect their ratings
- val partialInInfos = ratings.map { x => (x._1, x._2.item, x._2.user, x._2.rating)}
- .withForwardedFields("0").groupBy(0, 1).reduceGroup {
- x =>
- var userBlockID = -1
- var itemID = -1L
- val userIDs = ArrayBuffer[Long]()
- val ratings = ArrayBuffer[Double]()
-
- while (x.hasNext) {
- val (uBlockID, item, user, rating) = x.next
- userBlockID = uBlockID
- itemID = item
-
- userIDs += user
- ratings += rating
- }
-
- (userBlockID, blockIDGenerator(itemID), itemID, (userIDs.toArray, ratings.toArray))
- }.withForwardedFields("0")
-
- // Aggregate all ratings for items belonging to the same item block. Sort ascending with
- // respect to the itemID, because later the item vectors of the update message are sorted
- // accordingly.
- val collectedPartialInfos = partialInInfos.groupBy(0, 1).sortGroup(2, Order.ASCENDING).
- reduceGroup {
- new GroupReduceFunction[(Int, Int, Long, (Array[Long], Array[Double])), (Int,
- Int, Array[(Array[Long], Array[Double])])](){
- val buffer = new ArrayBuffer[(Array[Long], Array[Double])]
-
- override def reduce(iterable: lang.Iterable[(Int, Int, Long, (Array[Long],
- Array[Double]))], collector: Collector[(Int, Int, Array[(Array[Long],
- Array[Double])])]): Unit = {
-
- val infos = iterable.iterator()
- var counter = 0
-
- var blockID = -1
- var itemBlockID = -1
-
- while (infos.hasNext && counter < buffer.length) {
- val info = infos.next()
- blockID = info._1
- itemBlockID = info._2
-
- buffer(counter) = info._4
-
- counter += 1
- }
-
- while (infos.hasNext) {
- val info = infos.next()
- blockID = info._1
- itemBlockID = info._2
-
- buffer += info._4
-
- counter += 1
- }
-
- val array = new Array[(Array[Long], Array[Double])](counter)
-
- buffer.copyToArray(array)
-
- collector.collect((blockID, itemBlockID, array))
- }
- }
- }.withForwardedFields("0", "1")
-
- // Aggregate all item block ratings with respect to their user block ID. Sort the blocks with
- // respect to their itemBlockID, because the block update messages are sorted the same way
- collectedPartialInfos.coGroup(usersPerBlock).where(0).equalTo(0).
- sortFirstGroup(1, Order.ASCENDING).apply{
- new CoGroupFunction[(Int, Int, Array[(Array[Long], Array[Double])]),
- (Int, Array[Long]), (Int, InBlockInformation)] {
- val buffer = ArrayBuffer[BlockRating]()
-
- override def coGroup(partialInfosIterable:
- lang.Iterable[(Int, Int, Array[(Array[Long], Array[Double])])],
- userIterable: lang.Iterable[(Int, Array[Long])],
- collector: Collector[(Int, InBlockInformation)]): Unit = {
-
- val users = userIterable.iterator()
- val partialInfos = partialInfosIterable.iterator()
-
- val userWrapper = users.next()
- val id = userWrapper._1
- val userIDs = userWrapper._2
- val userIDToPos = userIDs.zipWithIndex.toMap
-
- var counter = 0
-
- while (partialInfos.hasNext && counter < buffer.length) {
- val partialInfo = partialInfos.next()
- // entry contains the ratings and userIDs of a complete item block
- val entry = partialInfo._3
-
- val blockRelativeIndicesRatings = new Array[(Array[Int], Array[Double])](entry.size)
-
- // transform userIDs to positional indices
- for (row <- 0 until entry.length) {
- val rowEntries = entry(row)._1
- val rowIndices = new Array[Int](rowEntries.length)
-
- for (col <- 0 until rowEntries.length) {
- rowIndices(col) = userIDToPos(rowEntries(col))
- }
-
- blockRelativeIndicesRatings(row) = (rowIndices, entry(row)._2)
- }
-
- buffer(counter).ratings = blockRelativeIndicesRatings
-
- counter += 1
- }
-
- while (partialInfos.hasNext) {
- val partialInfo = partialInfos.next()
- // entry contains the ratings and userIDs of a complete item block
- val entry = partialInfo._3
- val blockRelativeIndicesRatings = new Array[(Array[Int], Array[Double])](entry.size)
-
- // transform userIDs to positional indices
- for (row <- 0 until entry.length) {
- val rowEntries = entry(row)._1
- val rowIndices = new Array[Int](rowEntries.length)
-
- for (col <- 0 until rowEntries.length) {
- rowIndices(col) = userIDToPos(rowEntries(col))
- }
-
- blockRelativeIndicesRatings(row) = (rowIndices, entry(row)._2)
- }
-
- buffer += new BlockRating(blockRelativeIndicesRatings)
... 4430 lines suppressed ...