You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by uc...@apache.org on 2015/06/30 12:15:57 UTC
[27/51] [partial] flink-web git commit: [hotfix] Manual build of docs
http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/quickstart.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/quickstart.html b/content/docs/0.9/libs/ml/quickstart.html
new file mode 100644
index 0000000..188864b
--- /dev/null
+++ b/content/docs/0.9/libs/ml/quickstart.html
@@ -0,0 +1,438 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+
+ <title>Apache Flink 0.9.0 Documentation: FlinkML - Quickstart Guide</title>
+
+ <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+ <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+ <!-- Bootstrap -->
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+
+ <script type="text/x-mathjax-config">
+ MathJax.Hub.Config({
+ tex2jax: {
+ inlineMath: [['$','$'], ['\\(','\\)']] },
+ TeX: {
+ equationNumbers: { autoNumber: "AMS" } }
+ });
+ </script>
+ <script type="text/javascript"
+ src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
+ </script>
+
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ </head>
+ <body>
+
+
+
+
+
+
+ <!-- Top navbar. -->
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <!-- The logo. -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <div class="navbar-logo">
+ <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+ </div>
+ </div><!-- /.navbar-header -->
+
+ <!-- The navigation links. -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+ <!-- Setup -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+ </ul>
+ </li>
+
+ <!-- Programming Guides -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Libraries -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Internals -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture & Process Model</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction & Serialization</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs & Scheduling</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+ </ul>
+ </li>
+ </ul>
+ <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+ <div class="form-group">
+ <input type="text" class="form-control" name="q" placeholder="Search all pages">
+ </div>
+ <button type="submit" class="btn btn-default">Search</button>
+ </form>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container -->
+ </nav>
+
+
+
+
+
+<!--Some of the Latex math notation has been adapted from Apache Spark MLlib's documentation-->
+$$
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\x}{\mathbf{x}}
+\newcommand{\y}{\mathbf{y}}
+\newcommand{\wv}{\mathbf{w}}
+\newcommand{\av}{\mathbf{\alpha}}
+\newcommand{\bv}{\mathbf{b}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\id}{\mathbf{I}}
+\newcommand{\ind}{\mathbf{1}}
+\newcommand{\0}{\mathbf{0}}
+\newcommand{\unit}{\mathbf{e}}
+\newcommand{\one}{\mathbf{1}}
+\newcommand{\zero}{\mathbf{0}}
+\newcommand\rfrac[2]{^{#1}\!/_{#2}}
+\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
+$$
+
+
+ <!-- Main content. -->
+ <div class="container">
+
+
+<div class="row">
+ <div class="col-sm-10 col-sm-offset-1">
+ <h1><a href="../ml">FlinkML</a> - Quickstart Guide</h1>
+
+
+
+<ul id="markdown-toc">
+ <li><a href="#introduction" id="markdown-toc-introduction">Introduction</a></li>
+ <li><a href="#linking-with-flinkml" id="markdown-toc-linking-with-flinkml">Linking with FlinkML</a></li>
+ <li><a href="#loading-data" id="markdown-toc-loading-data">Loading data</a></li>
+ <li><a href="#classification" id="markdown-toc-classification">Classification</a></li>
+ <li><a href="#data-pre-processing-and-pipelines" id="markdown-toc-data-pre-processing-and-pipelines">Data pre-processing and pipelines</a></li>
+ <li><a href="#where-to-go-from-here" id="markdown-toc-where-to-go-from-here">Where to go from here</a></li>
+</ul>
+
+<h2 id="introduction">Introduction</h2>
+
+<p>FlinkML is designed to make learning from your data a straight-forward process, abstracting away
+the complexities that usually come with big data learning tasks. In this
+quick-start guide we will show just how easy it is to solve a simple supervised learning problem
+using FlinkML. But first some basics, feel free to skip the next few lines if you’re already
+familiar with Machine Learning (ML).</p>
+
+<p>As defined by Murphy <a href="#murphy">[1]</a> ML deals with detecting patterns in data, and using those
+learned patterns to make predictions about the future. We can categorize most ML algorithms into
+two major categories: Supervised and Unsupervised Learning.</p>
+
+<ul>
+ <li>
+ <p><strong>Supervised Learning</strong> deals with learning a function (mapping) from a set of inputs
+(features) to a set of outputs. The learning is done using a <em>training set</em> of (input,
+output) pairs that we use to approximate the mapping function. Supervised learning problems are
+further divided into classification and regression problems. In classification problems we try to
+predict the <em>class</em> that an example belongs to, for example whether a user is going to click on
+an ad or not. Regression problems one the other hand, are about predicting (real) numerical
+values, often called the dependent variable, for example what the temperature will be tomorrow.</p>
+ </li>
+ <li>
+ <p><strong>Unsupervised Learning</strong> deals with discovering patterns and regularities in the data. An example
+of this would be <em>clustering</em>, where we try to discover groupings of the data from the
+descriptive features. Unsupervised learning can also be used for feature selection, for example
+through <a href="https://en.wikipedia.org/wiki/Principal_component_analysis">principal components analysis</a>.</p>
+ </li>
+</ul>
+
+<h2 id="linking-with-flinkml">Linking with FlinkML</h2>
+
+<p>In order to use FlinkML in your project, first you have to
+<a href="http://ci.apache.org/projects/flink/flink-docs-master/apis/programming_guide.html#linking-with-flink">set up a Flink program</a>.
+Next, you have to add the FlinkML dependency to the <code>pom.xml</code> of your project:</p>
+
+<div class="highlight"><pre><code class="language-xml" data-lang="xml"><span class="nt"><dependency></span>
+ <span class="nt"><groupId></span>org.apache.flink<span class="nt"></groupId></span>
+ <span class="nt"><artifactId></span>flink-ml<span class="nt"></artifactId></span>
+ <span class="nt"><version></span>0.9.0<span class="nt"></version></span>
+<span class="nt"></dependency></span></code></pre></div>
+
+<h2 id="loading-data">Loading data</h2>
+
+<p>To load data to be used with FlinkML we can use the ETL capabilities of Flink, or specialized
+functions for formatted data, such as the LibSVM format. For supervised learning problems it is
+common to use the <code>LabeledVector</code> class to represent the <code>(label, features)</code> examples. A <code>LabeledVector</code>
+object will have a FlinkML <code>Vector</code> member representing the features of the example and a <code>Double</code>
+member which represents the label, which could be the class in a classification problem, or the dependent
+variable for a regression problem.</p>
+
+<p>As an example, we can use Haberman’s Survival Data Set , which you can
+<a href="http://archive.ics.uci.edu/ml/machine-learning-databases/haberman/haberman.data">download from the UCI ML repository</a>.
+This dataset <em>“contains cases from a study conducted on the survival of patients who had undergone
+surgery for breast cancer”</em>. The data comes in a comma-separated file, where the first 3 columns
+are the features and last column is the class, and the 4th column indicates whether the patient
+survived 5 years or longer (label 1), or died within 5 years (label 2). You can check the <a href="https://archive.ics.uci.edu/ml/datasets/Haberman%27s+Survival">UCI
+page</a> for more information on the data.</p>
+
+<p>We can load the data as a <code>DataSet[String]</code> first:</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.api.scala.ExecutionEnvironment</span>
+
+<span class="k">val</span> <span class="n">env</span> <span class="k">=</span> <span class="nc">ExecutionEnvironment</span><span class="o">.</span><span class="n">getExecutionEnvironment</span>
+
+<span class="k">val</span> <span class="n">survival</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">readCsvFile</span><span class="o">[(</span><span class="kt">String</span>, <span class="kt">String</span>, <span class="kt">String</span>, <span class="kt">String</span><span class="o">)](</span><span class="s">"/path/to/haberman.data"</span><span class="o">)</span></code></pre></div>
+
+<p>We can now transform the data into a <code>DataSet[LabeledVector]</code>. This will allow us to use the
+dataset with the FlinkML classification algorithms. We know that the 4th element of the dataset
+is the class label, and the rest are features, so we can build <code>LabeledVector</code> elements like this:</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.common.LabeledVector</span>
+<span class="k">import</span> <span class="nn">org.apache.flink.ml.math.DenseVector</span>
+
+<span class="k">val</span> <span class="n">survivalLV</span> <span class="k">=</span> <span class="n">survival</span>
+ <span class="o">.</span><span class="n">map</span><span class="o">{</span><span class="n">tuple</span> <span class="k">=></span>
+ <span class="k">val</span> <span class="n">list</span> <span class="k">=</span> <span class="n">tuple</span><span class="o">.</span><span class="n">productIterator</span><span class="o">.</span><span class="n">toList</span>
+ <span class="k">val</span> <span class="n">numList</span> <span class="k">=</span> <span class="n">list</span><span class="o">.</span><span class="n">map</span><span class="o">(</span><span class="k">_</span><span class="o">.</span><span class="n">asInstanceOf</span><span class="o">[</span><span class="kt">String</span><span class="o">].</span><span class="n">toDouble</span><span class="o">)</span>
+ <span class="nc">LabeledVector</span><span class="o">(</span><span class="n">numList</span><span class="o">(</span><span class="mi">3</span><span class="o">),</span> <span class="nc">DenseVector</span><span class="o">(</span><span class="n">numList</span><span class="o">.</span><span class="n">take</span><span class="o">(</span><span class="mi">3</span><span class="o">).</span><span class="n">toArray</span><span class="o">))</span>
+ <span class="o">}</span></code></pre></div>
+
+<p>We can then use this data to train a learner. We will however use another dataset to exemplify
+building a learner; that will allow us to show how we can import other dataset formats.</p>
+
+<p><strong>LibSVM files</strong></p>
+
+<p>A common format for ML datasets is the LibSVM format and a number of datasets using that format can be
+found <a href="http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/">in the LibSVM datasets website</a>. FlinkML provides utilities for loading
+datasets using the LibSVM format through the <code>readLibSVM</code> function available through the <code>MLUtils</code>
+object.
+You can also save datasets in the LibSVM format using the <code>writeLibSVM</code> function.
+Let’s import the svmguide1 dataset. You can download the
+<a href="http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/svmguide1">training set here</a>
+and the <a href="http://www.csie.ntu.edu.tw/~cjlin/libsvmtools/datasets/binary/svmguide1.t">test set here</a>.
+This is an astroparticle binary classification dataset, used by Hsu et al. <a href="#hsu">[3]</a> in their
+practical Support Vector Machine (SVM) guide. It contains 4 numerical features, and the class label.</p>
+
+<p>We can simply import the dataset then using:</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.MLUtils</span>
+
+<span class="k">val</span> <span class="n">astroTrain</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="nc">MLUtils</span><span class="o">.</span><span class="n">readLibSVM</span><span class="o">(</span><span class="s">"/path/to/svmguide1"</span><span class="o">)</span>
+<span class="k">val</span> <span class="n">astroTest</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="nc">MLUtils</span><span class="o">.</span><span class="n">readLibSVM</span><span class="o">(</span><span class="s">"/path/to/svmguide1.t"</span><span class="o">)</span></code></pre></div>
+
+<p>This gives us two <code>DataSet[LabeledVector]</code> objects that we will use in the following section to
+create a classifier.</p>
+
+<h2 id="classification">Classification</h2>
+
+<p>Once we have imported the dataset we can train a <code>Predictor</code> such as a linear SVM classifier.
+We can set a number of parameters for the classifier. Here we set the <code>Blocks</code> parameter,
+which is used to split the input by the underlying CoCoA algorithm <a href="#jaggi">[2]</a> uses. The
+regularization parameter determines the amount of $l_2$ regularization applied, which is used
+to avoid overfitting. The step size determines the contribution of the weight vector updates to
+the next weight vector value. This parameter sets the initial step size.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.classification.SVM</span>
+
+<span class="k">val</span> <span class="n">svm</span> <span class="k">=</span> <span class="nc">SVM</span><span class="o">()</span>
+ <span class="o">.</span><span class="n">setBlocks</span><span class="o">(</span><span class="n">env</span><span class="o">.</span><span class="n">getParallelism</span><span class="o">)</span>
+ <span class="o">.</span><span class="n">setIterations</span><span class="o">(</span><span class="mi">100</span><span class="o">)</span>
+ <span class="o">.</span><span class="n">setRegularization</span><span class="o">(</span><span class="mf">0.001</span><span class="o">)</span>
+ <span class="o">.</span><span class="n">setStepsize</span><span class="o">(</span><span class="mf">0.1</span><span class="o">)</span>
+ <span class="o">.</span><span class="n">setSeed</span><span class="o">(</span><span class="mi">42</span><span class="o">)</span>
+
+<span class="n">svm</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">astroTrain</span><span class="o">)</span></code></pre></div>
+
+<p>We can now make predictions on the test set.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">val</span> <span class="n">predictionPairs</span> <span class="k">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">astroTest</span><span class="o">)</span></code></pre></div>
+
+<p>Next we will see how we can pre-process our data, and use the ML pipelines capabilities of FlinkML.</p>
+
+<h2 id="data-pre-processing-and-pipelines">Data pre-processing and pipelines</h2>
+
+<p>A pre-processing step that is often encouraged <a href="#hsu">[3]</a> when using SVM classification is scaling
+the input features to the [0, 1] range, in order to avoid features with extreme values
+dominating the rest.
+FlinkML has a number of <code>Transformers</code> such as <code>MinMaxScaler</code> that are used to pre-process data,
+and a key feature is the ability to chain <code>Transformers</code> and <code>Predictors</code> together. This allows
+us to run the same pipeline of transformations and make predictions on the train and test data in
+a straight-forward and type-safe manner. You can read more on the pipeline system of FlinkML
+<a href="pipelines.html">in the pipelines documentation</a>.</p>
+
+<p>Let us first create a normalizing transformer for the features in our dataset, and chain it to a
+new SVM classifier.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="k">import</span> <span class="nn">org.apache.flink.ml.preprocessing.MinMaxScaler</span>
+
+<span class="k">val</span> <span class="n">scaler</span> <span class="k">=</span> <span class="nc">MinMaxScaler</span><span class="o">()</span>
+
+<span class="k">val</span> <span class="n">scaledSVM</span> <span class="k">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">chainPredictor</span><span class="o">(</span><span class="n">svm</span><span class="o">)</span></code></pre></div>
+
+<p>We can now use our newly created pipeline to make predictions on the test set.
+First we call fit again, to train the scaler and the SVM classifier.
+The data of the test set will then be automatically scaled before being passed on to the SVM to
+make predictions.</p>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="n">scaledSVM</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">astroTrain</span><span class="o">)</span>
+
+<span class="k">val</span> <span class="n">predictionPairsScaled</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[(</span><span class="kt">Double</span>, <span class="kt">Double</span><span class="o">)]</span> <span class="k">=</span> <span class="n">scaledSVM</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">astroTest</span><span class="o">)</span></code></pre></div>
+
+<p>The scaled inputs should give us better prediction performance.
+The result of the prediction on <code>LabeledVector</code>s is a data set of tuples where the first entry denotes the true label value and the second entry is the predicted label value.</p>
+
+<h2 id="where-to-go-from-here">Where to go from here</h2>
+
+<p>This quickstart guide can act as an introduction to the basic concepts of FlinkML, but there’s a lot
+more you can do.
+We recommend going through the <a href="index.html">FlinkML documentation</a>, and trying out the different
+algorithms.
+A very good way to get started is to play around with interesting datasets from the UCI ML
+repository and the LibSVM datasets.
+Tackling an interesting problem from a website like <a href="https://www.kaggle.com">Kaggle</a> or
+<a href="http://www.drivendata.org/">DrivenData</a> is also a great way to learn by competing with other
+data scientists.
+If you would like to contribute some new algorithms take a look at our
+<a href="contribution_guide.html">contribution guide</a>.</p>
+
+<p><strong>References</strong></p>
+
+<p><a name="murphy"></a>[1] Murphy, Kevin P. <em>Machine learning: a probabilistic perspective.</em> MIT
+press, 2012.</p>
+
+<p><a name="jaggi"></a>[2] Jaggi, Martin, et al. <em>Communication-efficient distributed dual
+coordinate ascent.</em> Advances in Neural Information Processing Systems. 2014.</p>
+
+<p><a name="hsu"></a>[3] Hsu, Chih-Wei, Chih-Chung Chang, and Chih-Jen Lin.
+ <em>A practical guide to support vector classification.</em> 2003.</p>
+
+ </div>
+
+ <div class="col-sm-10 col-sm-offset-1">
+ <!-- Disqus thread and some vertical offset -->
+ <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+ </div>
+</div>
+
+ </div><!-- /.container -->
+
+ <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+ <!-- Include all compiled plugins (below), or include individual files as needed -->
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+ <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-52545728-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+
+ <!-- Disqus -->
+ <script type="text/javascript">
+ var disqus_shortname = 'stratosphere-eu';
+ (function() {
+ var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+ dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+ (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+ })();
+</script>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/standard_scaler.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/standard_scaler.html b/content/docs/0.9/libs/ml/standard_scaler.html
new file mode 100644
index 0000000..9f84bfa
--- /dev/null
+++ b/content/docs/0.9/libs/ml/standard_scaler.html
@@ -0,0 +1,340 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+
+ <title>Apache Flink 0.9.0 Documentation: FlinkML - Standard Scaler</title>
+
+ <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+ <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+ <!-- Bootstrap -->
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+
+ <script type="text/x-mathjax-config">
+ MathJax.Hub.Config({
+ tex2jax: {
+ inlineMath: [['$','$'], ['\\(','\\)']] },
+ TeX: {
+ equationNumbers: { autoNumber: "AMS" } }
+ });
+ </script>
+ <script type="text/javascript"
+ src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
+ </script>
+
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ </head>
+ <body>
+
+
+
+
+
+
+ <!-- Top navbar. -->
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <!-- The logo. -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <div class="navbar-logo">
+ <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+ </div>
+ </div><!-- /.navbar-header -->
+
+ <!-- The navigation links. -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+ <!-- Setup -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+ </ul>
+ </li>
+
+ <!-- Programming Guides -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Libraries -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Internals -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture & Process Model</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction & Serialization</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs & Scheduling</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+ </ul>
+ </li>
+ </ul>
+ <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+ <div class="form-group">
+ <input type="text" class="form-control" name="q" placeholder="Search all pages">
+ </div>
+ <button type="submit" class="btn btn-default">Search</button>
+ </form>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container -->
+ </nav>
+
+
+
+
+
+<!--Some of the Latex math notation has been adapted from Apache Spark MLlib's documentation-->
+$$
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\x}{\mathbf{x}}
+\newcommand{\y}{\mathbf{y}}
+\newcommand{\wv}{\mathbf{w}}
+\newcommand{\av}{\mathbf{\alpha}}
+\newcommand{\bv}{\mathbf{b}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\id}{\mathbf{I}}
+\newcommand{\ind}{\mathbf{1}}
+\newcommand{\0}{\mathbf{0}}
+\newcommand{\unit}{\mathbf{e}}
+\newcommand{\one}{\mathbf{1}}
+\newcommand{\zero}{\mathbf{0}}
+\newcommand\rfrac[2]{^{#1}\!/_{#2}}
+\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
+$$
+
+
+ <!-- Main content. -->
+ <div class="container">
+
+
+<div class="row">
+ <div class="col-sm-10 col-sm-offset-1">
+ <h1><a href="../ml">FlinkML</a> - Standard Scaler</h1>
+
+
+
+<ul id="markdown-toc">
+ <li><a href="#description" id="markdown-toc-description">Description</a></li>
+ <li><a href="#operations" id="markdown-toc-operations">Operations</a> <ul>
+ <li><a href="#fit" id="markdown-toc-fit">Fit</a></li>
+ <li><a href="#transform" id="markdown-toc-transform">Transform</a></li>
+ </ul>
+ </li>
+ <li><a href="#parameters" id="markdown-toc-parameters">Parameters</a></li>
+ <li><a href="#examples" id="markdown-toc-examples">Examples</a></li>
+</ul>
+
+<h2 id="description">Description</h2>
+
+<p>The standard scaler scales the given data set, so that all features will have a user specified mean and variance.
+ In case the user does not provide a specific mean and standard deviation, the standard scaler transforms the features of the input data set to have mean equal to 0 and standard deviation equal to 1.
+ Given a set of input data $x_1, x_2,… x_n$, with mean:</p>
+
+<script type="math/tex; mode=display">\bar{x} = \frac{1}{n}\sum_{i=1}^{n}x_{i}</script>
+
+<p>and standard deviation:</p>
+
+<script type="math/tex; mode=display">\sigma_{x}=\sqrt{ \frac{1}{n} \sum_{i=1}^{n}(x_{i}-\bar{x})^{2}}</script>
+
+<p>The scaled data set $z_1, z_2,…,z_n$ will be:</p>
+
+<script type="math/tex; mode=display">z_{i}= std \left (\frac{x_{i} - \bar{x} }{\sigma_{x}}\right ) + mean</script>
+
+<p>where $\textit{std}$ and $\textit{mean}$ are the user specified values for the standard deviation and mean.</p>
+
+<h2 id="operations">Operations</h2>
+
+<p><code>StandardScaler</code> is a <code>Transformer</code>.
+As such, it supports the <code>fit</code> and <code>transform</code> operation.</p>
+
+<h3 id="fit">Fit</h3>
+
+<p>StandardScaler is trained on all subtypes of <code>Vector</code> or <code>LabeledVector</code>:</p>
+
+<ul>
+ <li><code>fit[T <: Vector]: DataSet[T] => Unit</code></li>
+ <li><code>fit: DataSet[LabeledVector] => Unit</code></li>
+</ul>
+
+<h3 id="transform">Transform</h3>
+
+<p>StandardScaler transforms all subtypes of <code>Vector</code> or <code>LabeledVector</code> into the respective type:</p>
+
+<ul>
+ <li><code>transform[T <: Vector]: DataSet[T] => DataSet[T]</code></li>
+ <li><code>transform: DataSet[LabeledVector] => DataSet[LabeledVector]</code></li>
+</ul>
+
+<h2 id="parameters">Parameters</h2>
+
+<p>The standard scaler implementation can be controlled by the following two parameters:</p>
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Parameters</th>
+ <th class="text-center">Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td><strong>Mean</strong></td>
+ <td>
+ <p>
+ The mean of the scaled data set. (Default value: <strong>0.0</strong>)
+ </p>
+ </td>
+ </tr>
+ <tr>
+ <td><strong>Std</strong></td>
+ <td>
+ <p>
+ The standard deviation of the scaled data set. (Default value: <strong>1.0</strong>)
+ </p>
+ </td>
+ </tr>
+ </tbody>
+</table>
+
+<h2 id="examples">Examples</h2>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="c1">// Create standard scaler transformer</span>
+<span class="k">val</span> <span class="n">scaler</span> <span class="k">=</span> <span class="nc">StandardScaler</span><span class="o">()</span>
+<span class="o">.</span><span class="n">setMean</span><span class="o">(</span><span class="mf">10.0</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setStd</span><span class="o">(</span><span class="mf">2.0</span><span class="o">)</span>
+
+<span class="c1">// Obtain data set to be scaled</span>
+<span class="k">val</span> <span class="n">dataSet</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="o">...</span>
+
+<span class="c1">// Learn the mean and standard deviation of the training data</span>
+<span class="n">scaler</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">dataSet</span><span class="o">)</span>
+
+<span class="c1">// Scale the provided data set to have mean=10.0 and std=2.0</span>
+<span class="k">val</span> <span class="n">scaledDS</span> <span class="k">=</span> <span class="n">scaler</span><span class="o">.</span><span class="n">transform</span><span class="o">(</span><span class="n">dataSet</span><span class="o">)</span></code></pre></div>
+
+
+ </div>
+
+ <div class="col-sm-10 col-sm-offset-1">
+ <!-- Disqus thread and some vertical offset -->
+ <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+ </div>
+</div>
+
+ </div><!-- /.container -->
+
+ <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+ <!-- Include all compiled plugins (below), or include individual files as needed -->
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+ <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-52545728-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+
+ <!-- Disqus -->
+ <script type="text/javascript">
+ var disqus_shortname = 'stratosphere-eu';
+ (function() {
+ var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+ dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+ (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+ })();
+</script>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/svm.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/svm.html b/content/docs/0.9/libs/ml/svm.html
new file mode 100644
index 0000000..00184c4
--- /dev/null
+++ b/content/docs/0.9/libs/ml/svm.html
@@ -0,0 +1,419 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+
+ <title>Apache Flink 0.9.0 Documentation: FlinkML - SVM using CoCoA</title>
+
+ <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+ <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+ <!-- Bootstrap -->
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+
+ <script type="text/x-mathjax-config">
+ MathJax.Hub.Config({
+ tex2jax: {
+ inlineMath: [['$','$'], ['\\(','\\)']] },
+ TeX: {
+ equationNumbers: { autoNumber: "AMS" } }
+ });
+ </script>
+ <script type="text/javascript"
+ src="https://cdn.mathjax.org/mathjax/latest/MathJax.js?config=TeX-AMS-MML_HTMLorMML">
+ </script>
+
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ </head>
+ <body>
+
+
+
+
+
+
+ <!-- Top navbar. -->
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <!-- The logo. -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <div class="navbar-logo">
+ <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+ </div>
+ </div><!-- /.navbar-header -->
+
+ <!-- The navigation links. -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+ <!-- Setup -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+ </ul>
+ </li>
+
+ <!-- Programming Guides -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Libraries -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Internals -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture & Process Model</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction & Serialization</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs & Scheduling</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+ </ul>
+ </li>
+ </ul>
+ <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+ <div class="form-group">
+ <input type="text" class="form-control" name="q" placeholder="Search all pages">
+ </div>
+ <button type="submit" class="btn btn-default">Search</button>
+ </form>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container -->
+ </nav>
+
+
+
+
+
+<!--Some of the Latex math notation has been adapted from Apache Spark MLlib's documentation-->
+$$
+\newcommand{\R}{\mathbb{R}}
+\newcommand{\E}{\mathbb{E}}
+\newcommand{\x}{\mathbf{x}}
+\newcommand{\y}{\mathbf{y}}
+\newcommand{\wv}{\mathbf{w}}
+\newcommand{\av}{\mathbf{\alpha}}
+\newcommand{\bv}{\mathbf{b}}
+\newcommand{\N}{\mathbb{N}}
+\newcommand{\id}{\mathbf{I}}
+\newcommand{\ind}{\mathbf{1}}
+\newcommand{\0}{\mathbf{0}}
+\newcommand{\unit}{\mathbf{e}}
+\newcommand{\one}{\mathbf{1}}
+\newcommand{\zero}{\mathbf{0}}
+\newcommand\rfrac[2]{^{#1}\!/_{#2}}
+\newcommand{\norm}[1]{\left\lVert#1\right\rVert}
+$$
+
+
+ <!-- Main content. -->
+ <div class="container">
+
+
+<div class="row">
+ <div class="col-sm-10 col-sm-offset-1">
+ <h1><a href="../ml">FlinkML</a> - SVM using CoCoA</h1>
+
+
+
+<ul id="markdown-toc">
+ <li><a href="#description" id="markdown-toc-description">Description</a></li>
+ <li><a href="#operations" id="markdown-toc-operations">Operations</a> <ul>
+ <li><a href="#fit" id="markdown-toc-fit">Fit</a></li>
+ <li><a href="#predict" id="markdown-toc-predict">Predict</a></li>
+ </ul>
+ </li>
+ <li><a href="#parameters" id="markdown-toc-parameters">Parameters</a></li>
+ <li><a href="#examples" id="markdown-toc-examples">Examples</a></li>
+</ul>
+
+<h2 id="description">Description</h2>
+
+<p>Implements an SVM with soft-margin using the communication-efficient distributed dual coordinate
+ascent algorithm with hinge-loss function.
+The algorithm solves the following minimization problem:</p>
+
+<script type="math/tex; mode=display">\min_{\mathbf{w} \in \mathbb{R}^d} \frac{\lambda}{2} \left\lVert \mathbf{w} \right\rVert^2 + \frac{1}{n} \sum_{i=1}^n l_{i}\left(\mathbf{w}^T\mathbf{x}_i\right)</script>
+
+<p>with $\mathbf{w}$ being the weight vector, $\lambda$ being the regularization constant,
+<script type="math/tex">\mathbf{x}_i \in \mathbb{R}^d</script> being the data points and <script type="math/tex">l_{i}</script> being the convex loss
+functions, which can also depend on the labels <script type="math/tex">y_{i} \in \mathbb{R}</script>.
+In the current implementation the regularizer is the $\ell_2$-norm and the loss functions are the hinge-loss functions:</p>
+
+<script type="math/tex; mode=display">l_{i} = \max\left(0, 1 - y_{i} \mathbf{w}^T\mathbf{x}_i \right)</script>
+
+<p>With these choices, the problem definition is equivalent to a SVM with soft-margin.
+Thus, the algorithm allows us to train a SVM with soft-margin.</p>
+
+<p>The minimization problem is solved by applying stochastic dual coordinate ascent (SDCA).
+In order to make the algorithm efficient in a distributed setting, the CoCoA algorithm calculates
+several iterations of SDCA locally on a data block before merging the local updates into a
+valid global state.
+This state is redistributed to the different data partitions where the next round of local SDCA
+iterations is then executed.
+The number of outer iterations and local SDCA iterations control the overall network costs, because
+there is only network communication required for each outer iteration.
+The local SDCA iterations are embarrassingly parallel once the individual data partitions have been
+distributed across the cluster.</p>
+
+<p>The implementation of this algorithm is based on the work of
+<a href="http://arxiv.org/abs/1409.1458">Jaggi et al.</a></p>
+
+<h2 id="operations">Operations</h2>
+
+<p><code>SVM</code> is a <code>Predictor</code>.
+As such, it supports the <code>fit</code> and <code>predict</code> operation.</p>
+
+<h3 id="fit">Fit</h3>
+
+<p>SVM is trained given a set of <code>LabeledVector</code>:</p>
+
+<ul>
+ <li><code>fit: DataSet[LabeledVector] => Unit</code></li>
+</ul>
+
+<h3 id="predict">Predict</h3>
+
+<p>SVM predicts for all subtypes of <code>Vector</code> the corresponding class label:</p>
+
+<ul>
+ <li><code>predict[T <: Vector]: DataSet[T] => DataSet[LabeledVector]</code></li>
+</ul>
+
+<p>If we call predict with a <code>DataSet[LabeledVector]</code>, we make a prediction on the class label
+for each example, and return a <code>DataSet[(Double, Double)]</code>. In each tuple the first element
+is the true value, as was provided from the input <code>DataSet[LabeledVector]</code> and the second element
+is the predicted value. You can then use these <code>(truth, prediction)</code> tuples to evaluate
+the algorithm’s performance.</p>
+
+<ul>
+ <li><code>predict: DataSet[LabeledVector] => DataSet[(Double, Double)]</code></li>
+</ul>
+
+<h2 id="parameters">Parameters</h2>
+
+<p>The SVM implementation can be controlled by the following parameters:</p>
+
+<table class="table table-bordered">
+ <thead>
+ <tr>
+ <th class="text-left" style="width: 20%">Parameters</th>
+ <th class="text-center">Description</th>
+ </tr>
+ </thead>
+
+ <tbody>
+ <tr>
+ <td><strong>Blocks</strong></td>
+ <td>
+ <p>
+ Sets the number of blocks into which the input data will be split.
+ On each block the local stochastic dual coordinate ascent method is executed.
+ This number should be set at least to the degree of parallelism.
+ If no value is specified, then the parallelism of the input DataSet is used as the number of blocks.
+ (Default value: <strong>None</strong>)
+ </p>
+ </td>
+ </tr>
+ <tr>
+ <td><strong>Iterations</strong></td>
+ <td>
+ <p>
+ Defines the maximum number of iterations of the outer loop method.
+ In other words, it defines how often the SDCA method is applied to the blocked data.
+ After each iteration, the locally computed weight vector updates have to be reduced to update the global weight vector value.
+ The new weight vector is broadcast to all SDCA tasks at the beginning of each iteration.
+ (Default value: <strong>10</strong>)
+ </p>
+ </td>
+ </tr>
+ <tr>
+ <td><strong>LocalIterations</strong></td>
+ <td>
+ <p>
+ Defines the maximum number of SDCA iterations.
+ In other words, it defines how many data points are drawn from each local data block to calculate the stochastic dual coordinate ascent.
+ (Default value: <strong>10</strong>)
+ </p>
+ </td>
+ </tr>
+ <tr>
+ <td><strong>Regularization</strong></td>
+ <td>
+ <p>
+ Defines the regularization constant of the SVM algorithm.
+ The higher the value, the smaller will the 2-norm of the weight vector be.
+ In case of a SVM with hinge loss this means that the SVM margin will be wider even though it might contain some false classifications.
+ (Default value: <strong>1.0</strong>)
+ </p>
+ </td>
+ </tr>
+ <tr>
+ <td><strong>Stepsize</strong></td>
+ <td>
+ <p>
+ Defines the initial step size for the updates of the weight vector.
+ The larger the step size is, the larger will be the contribution of the weight vector updates to the next weight vector value.
+ The effective scaling of the updates is $\frac{stepsize}{blocks}$.
+ This value has to be tuned in case that the algorithm becomes unstable.
+ (Default value: <strong>1.0</strong>)
+ </p>
+ </td>
+ </tr>
+ <tr>
+ <td><strong>Seed</strong></td>
+ <td>
+ <p>
+ Defines the seed to initialize the random number generator.
+ The seed directly controls which data points are chosen for the SDCA method.
+ (Default value: <strong>0</strong>)
+ </p>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+
+<h2 id="examples">Examples</h2>
+
+<div class="highlight"><pre><code class="language-scala" data-lang="scala"><span class="c1">// Read the training data set, from a LibSVM formatted file</span>
+<span class="k">val</span> <span class="n">trainingDS</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">readLibSVM</span><span class="o">(</span><span class="n">pathToTrainingFile</span><span class="o">)</span>
+
+<span class="c1">// Create the SVM learner</span>
+<span class="k">val</span> <span class="n">svm</span> <span class="k">=</span> <span class="nc">SVM</span><span class="o">()</span>
+<span class="o">.</span><span class="n">setBlocks</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setIterations</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setLocalIterations</span><span class="o">(</span><span class="mi">10</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setRegularization</span><span class="o">(</span><span class="mf">0.5</span><span class="o">)</span>
+<span class="o">.</span><span class="n">setStepsize</span><span class="o">(</span><span class="mf">0.5</span><span class="o">)</span>
+
+<span class="c1">// Learn the SVM model</span>
+<span class="n">svm</span><span class="o">.</span><span class="n">fit</span><span class="o">(</span><span class="n">trainingDS</span><span class="o">)</span>
+
+<span class="c1">// Read the testing data set</span>
+<span class="k">val</span> <span class="n">testingDS</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">Vector</span><span class="o">]</span> <span class="k">=</span> <span class="n">env</span><span class="o">.</span><span class="n">readVectorFile</span><span class="o">(</span><span class="n">pathToTestingFile</span><span class="o">)</span>
+
+<span class="c1">// Calculate the predictions for the testing data set</span>
+<span class="k">val</span> <span class="n">predictionDS</span><span class="k">:</span> <span class="kt">DataSet</span><span class="o">[</span><span class="kt">LabeledVector</span><span class="o">]</span> <span class="k">=</span> <span class="n">svm</span><span class="o">.</span><span class="n">predict</span><span class="o">(</span><span class="n">testingDS</span><span class="o">)</span></code></pre></div>
+
+
+ </div>
+
+ <div class="col-sm-10 col-sm-offset-1">
+ <!-- Disqus thread and some vertical offset -->
+ <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+ </div>
+</div>
+
+ </div><!-- /.container -->
+
+ <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+ <!-- Include all compiled plugins (below), or include individual files as needed -->
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+ <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-52545728-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+
+ <!-- Disqus -->
+ <script type="text/javascript">
+ var disqus_shortname = 'stratosphere-eu';
+ (function() {
+ var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+ dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+ (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+ })();
+</script>
+ </body>
+</html>
http://git-wip-us.apache.org/repos/asf/flink-web/blob/396616d4/content/docs/0.9/libs/ml/vision_roadmap.html
----------------------------------------------------------------------
diff --git a/content/docs/0.9/libs/ml/vision_roadmap.html b/content/docs/0.9/libs/ml/vision_roadmap.html
new file mode 100644
index 0000000..5793d68
--- /dev/null
+++ b/content/docs/0.9/libs/ml/vision_roadmap.html
@@ -0,0 +1,313 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE html>
+
+<html lang="en">
+ <head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1">
+ <!-- The above 3 meta tags *must* come first in the head; any other head content must come *after* these tags -->
+
+ <title>Apache Flink 0.9.0 Documentation: FlinkML - Vision and Roadmap</title>
+
+ <link rel="shortcut icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+ <link rel="icon" href="http://flink.apache.org/docs/0.9/page/favicon.ico" type="image/x-icon">
+
+ <!-- Bootstrap -->
+ <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/flink.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/syntax.css">
+ <link rel="stylesheet" href="http://flink.apache.org/docs/0.9/page/css/codetabs.css">
+
+ <!-- HTML5 shim and Respond.js for IE8 support of HTML5 elements and media queries -->
+ <!-- WARNING: Respond.js doesn't work if you view the page via file:// -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/html5shiv/3.7.2/html5shiv.min.js"></script>
+ <script src="https://oss.maxcdn.com/respond/1.4.2/respond.min.js"></script>
+ <![endif]-->
+ </head>
+ <body>
+
+
+
+
+
+
+ <!-- Top navbar. -->
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <!-- The logo. -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#bs-example-navbar-collapse-1">
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <div class="navbar-logo">
+ <a href="http://flink.apache.org"><img alt="Apache Flink" src="http://flink.apache.org/docs/0.9/page/img/navbar-brand-logo.jpg"></a>
+ </div>
+ </div><!-- /.navbar-header -->
+
+ <!-- The navigation links. -->
+ <div class="collapse navbar-collapse" id="bs-example-navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="http://flink.apache.org/docs/0.9/index.html">Overview<span class="hidden-sm hidden-xs"> 0.9.0</span></a></li>
+
+ <!-- Setup -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/setup" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Setup <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/setup/building.html">Get Flink 0.9-SNAPSHOT</a></li>
+
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Deployment</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/local_setup.html" class="active">Local</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/cluster_setup.html">Cluster (Standalone)</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/yarn_setup.html">YARN</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/gce_setup.html">GCloud</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/flink_on_tez.html">Flink on Tez <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="http://flink.apache.org/docs/0.9/setup/config.html">Configuration</a></li>
+ </ul>
+ </li>
+
+ <!-- Programming Guides -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/apis" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Programming Guides <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/apis/programming_guide.html"><strong>Batch: DataSet API</strong></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/streaming_guide.html"><strong>Streaming: DataStream API</strong> <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/python.html">Python API <span class="badge">Beta</span></a></li>
+
+ <li class="divider"></li>
+ <li><a href="scala_shell.html">Interactive Scala Shell</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/dataset_transformations.html">Dataset Transformations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/best_practices.html">Best Practices</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/example_connectors.html">Connectors</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/examples.html">Examples</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/local_execution.html">Local Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cluster_execution.html">Cluster Execution</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/cli.html">Command Line Interface</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/web_client.html">Web Client</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/iterations.html">Iterations</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/java8.html">Java 8</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/apis/hadoop_compatibility.html">Hadoop Compatability <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Libraries -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/libs" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Libraries <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li><a href="http://flink.apache.org/docs/0.9/libs/spargel_guide.html">Graphs: Spargel</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/gelly_guide.html">Graphs: Gelly <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/ml/">Machine Learning <span class="badge">Beta</span></a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/libs/table.html">Relational: Table <span class="badge">Beta</span></a></li>
+ </ul>
+ </li>
+
+ <!-- Internals -->
+ <li class="dropdown">
+ <a href="http://flink.apache.org/docs/0.9/internals" class="dropdown-toggle" data-toggle="dropdown" role="button" aria-expanded="false">Internals <span class="caret"></span></a>
+ <ul class="dropdown-menu" role="menu">
+ <li role="presentation" class="dropdown-header"><strong>Contribute</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/how_to_contribute.html">How to Contribute</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/coding_guidelines.html">Coding Guidelines</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/ide_setup.html">IDE Setup</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/logging.html">Logging</a></li>
+ <li class="divider"></li>
+ <li role="presentation" class="dropdown-header"><strong>Internals</strong></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/general_arch.html">Architecture & Process Model</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/types_serialization.html">Type Extraction & Serialization</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/job_scheduling.html">Jobs & Scheduling</a></li>
+ <li><a href="http://flink.apache.org/docs/0.9/internals/add_operator.html">How-To: Add an Operator</a></li>
+ </ul>
+ </li>
+ </ul>
+ <form class="navbar-form navbar-right hidden-sm hidden-md" role="search" action="http://flink.apache.org/docs/0.9/search-results.html">
+ <div class="form-group">
+ <input type="text" class="form-control" name="q" placeholder="Search all pages">
+ </div>
+ <button type="submit" class="btn btn-default">Search</button>
+ </form>
+ </div><!-- /.navbar-collapse -->
+ </div><!-- /.container -->
+ </nav>
+
+
+
+
+ <!-- Main content. -->
+ <div class="container">
+
+
+<div class="row">
+ <div class="col-sm-10 col-sm-offset-1">
+ <h1><a href="../ml">FlinkML</a> - Vision and Roadmap</h1>
+
+
+
+<ul id="markdown-toc">
+ <li><a href="#vision" id="markdown-toc-vision">Vision</a></li>
+ <li><a href="#roadmap" id="markdown-toc-roadmap">Roadmap</a></li>
+</ul>
+
+<h2 id="vision">Vision</h2>
+
+<p>The Machine Learning (ML) library for Flink is a new effort to bring scalable ML tools to the Flink
+community. Our goal is is to design and implement a system that is scalable and can deal with
+problems of various sizes, whether your data size is measured in megabytes or terabytes and beyond.
+We call this library FlinkML.</p>
+
+<p>An important concern for developers of ML systems is the amount of glue code that developers are
+forced to write [1] in the process of implementing an end-to-end ML system. Our goal with FlinkML
+is to help developers keep glue code to a minimum. The Flink ecosystem provides a great setting to
+tackle this problem, with its scalable ETL capabilities that can be easily combined inside the same
+program with FlinkML, allowing the development of robust pipelines without the need to use yet
+another technology for data ingestion and data munging.</p>
+
+<p>Another goal for FlinkML is to make the library easy to use. To that end we will be providing
+detailed documentation along with examples for every part of the system. Our aim is that developers
+will be able to get started with writing their ML pipelines quickly, using familiar programming
+concepts and terminology.</p>
+
+<p>Contrary to other data-processing systems, Flink exploits in-memory data streaming, and natively
+executes iterative processing algorithms which are common in ML. We plan to exploit the streaming
+nature of Flink, and provide functionality designed specifically for data streams.</p>
+
+<p>FlinkML will allow data scientists to test their models locally and using subsets of data, and then
+use the same code to run their algorithms at a much larger scale in a cluster setting.</p>
+
+<p>We are inspired by other open source efforts to provide ML systems, in particular
+<a href="http://scikit-learn.org/">scikit-learn</a> for cleanly specifying ML pipelines, and Spark’s
+<a href="https://spark.apache.org/mllib/">MLLib</a> for providing ML algorithms that scale with problem and
+cluster sizes.</p>
+
+<h2 id="roadmap">Roadmap</h2>
+
+<p>The roadmap below can provide an indication of the algorithms we aim to implement in the coming
+months. If you are interested in helping out, please check our <a href="contribution_guide.html">contribution guide</a>.
+Items in <strong>bold</strong> have already been implemented:</p>
+
+<ul>
+ <li>Pipelines of transformers and learners</li>
+ <li>Data pre-processing
+ <ul>
+ <li><strong>Feature scaling</strong></li>
+ <li><strong>Polynomial feature base mapper</strong></li>
+ <li>Feature hashing</li>
+ <li>Feature extraction for text</li>
+ <li>Dimensionality reduction</li>
+ </ul>
+ </li>
+ <li>Model selection and performance evaluation
+ <ul>
+ <li>Cross-validation for model selection and evaluation</li>
+ </ul>
+ </li>
+ <li>Supervised learning
+ <ul>
+ <li>Optimization framework
+ <ul>
+ <li><strong>Stochastic Gradient Descent</strong></li>
+ <li>L-BFGS</li>
+ </ul>
+ </li>
+ <li>Generalized Linear Models
+ <ul>
+ <li><strong>Multiple linear regression</strong></li>
+ <li>LASSO, Ridge regression</li>
+ <li>Multi-class Logistic regression</li>
+ </ul>
+ </li>
+ <li>Random forests</li>
+ <li><strong>Support Vector Machines</strong></li>
+ </ul>
+ </li>
+ <li>Unsupervised learning
+ <ul>
+ <li>Clustering
+ <ul>
+ <li>K-means clustering</li>
+ </ul>
+ </li>
+ <li>PCA</li>
+ </ul>
+ </li>
+ <li>Recommendation
+ <ul>
+ <li><strong>ALS</strong></li>
+ </ul>
+ </li>
+ <li>Text analytics
+ <ul>
+ <li>LDA</li>
+ </ul>
+ </li>
+ <li>Statistical estimation tools</li>
+ <li>Distributed linear algebra</li>
+ <li>Streaming ML</li>
+</ul>
+
+<p><strong>References:</strong></p>
+
+<p>[1] D. Sculley, G. Holt, D. Golovin, E. Davydov, T. Phillips, D. Ebner, V. Chaudhary,
+and M. Young. <em>Machine learning: The high interest credit card of technical debt.</em> In SE4ML:
+Software Engineering for Machine Learning (NIPS 2014 Workshop), 2014.</p>
+
+ </div>
+
+ <div class="col-sm-10 col-sm-offset-1">
+ <!-- Disqus thread and some vertical offset -->
+ <div style="margin-top: 75px; margin-bottom: 50px" id="disqus_thread"></div>
+ </div>
+</div>
+
+ </div><!-- /.container -->
+
+ <!-- jQuery (necessary for Bootstrap's JavaScript plugins) -->
+ <script src="https://ajax.googleapis.com/ajax/libs/jquery/1.11.2/jquery.min.js"></script>
+ <!-- Include all compiled plugins (below), or include individual files as needed -->
+ <script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js"></script>
+ <script src="http://flink.apache.org/docs/0.9/page/js/codetabs.js"></script>
+
+ <!-- Google Analytics -->
+ <script>
+ (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
+ (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
+ m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
+ })(window,document,'script','//www.google-analytics.com/analytics.js','ga');
+
+ ga('create', 'UA-52545728-1', 'auto');
+ ga('send', 'pageview');
+ </script>
+
+ <!-- Disqus -->
+ <script type="text/javascript">
+ var disqus_shortname = 'stratosphere-eu';
+ (function() {
+ var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
+ dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
+ (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
+ })();
+</script>
+ </body>
+</html>