You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@systemml.apache.org by de...@apache.org on 2017/04/06 20:19:08 UTC
svn commit: r1790452 - in /incubator/systemml/site: assets/css/main.css
documentation.html download.html get-started.html privacy-policy.html
Author: deron
Date: Thu Apr 6 20:19:08 2017
New Revision: 1790452
URL: http://svn.apache.org/viewvc?rev=1790452&view=rev
Log:
Add jekyll syntax highlighting to code blocks
Modified:
incubator/systemml/site/assets/css/main.css
incubator/systemml/site/documentation.html
incubator/systemml/site/download.html
incubator/systemml/site/get-started.html
incubator/systemml/site/privacy-policy.html
Modified: incubator/systemml/site/assets/css/main.css
URL: http://svn.apache.org/viewvc/incubator/systemml/site/assets/css/main.css?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/assets/css/main.css (original)
+++ incubator/systemml/site/assets/css/main.css Thu Apr 6 20:19:08 2017
@@ -4577,6 +4577,260 @@ input[type=number] {
font-weight: 700;
font-style: italic; }
+.highlight {
+ background: #ffffff; }
+
+.highlight .c {
+ color: #999988;
+ font-style: italic; }
+
+/* Comment */
+.highlight .err {
+ color: #a61717;
+ background-color: #e3d2d2; }
+
+/* Error */
+.highlight .k {
+ font-weight: bold; }
+
+/* Keyword */
+.highlight .o {
+ font-weight: bold; }
+
+/* Operator */
+.highlight .cm {
+ color: #999988;
+ font-style: italic; }
+
+/* Comment.Multiline */
+.highlight .cp {
+ color: #999999;
+ font-weight: bold; }
+
+/* Comment.Preproc */
+.highlight .c1 {
+ color: #999988;
+ font-style: italic; }
+
+/* Comment.Single */
+.highlight .cs {
+ color: #999999;
+ font-weight: bold;
+ font-style: italic; }
+
+/* Comment.Special */
+.highlight .gd {
+ color: #000000;
+ background-color: #ffdddd; }
+
+/* Generic.Deleted */
+.highlight .gd .x {
+ color: #000000;
+ background-color: #ffaaaa; }
+
+/* Generic.Deleted.Specific */
+.highlight .ge {
+ font-style: italic; }
+
+/* Generic.Emph */
+.highlight .gr {
+ color: #aa0000; }
+
+/* Generic.Error */
+.highlight .gh {
+ color: #999999; }
+
+/* Generic.Heading */
+.highlight .gi {
+ color: #000000;
+ background-color: #ddffdd; }
+
+/* Generic.Inserted */
+.highlight .gi .x {
+ color: #000000;
+ background-color: #aaffaa; }
+
+/* Generic.Inserted.Specific */
+.highlight .go {
+ color: #888888; }
+
+/* Generic.Output */
+.highlight .gp {
+ color: #555555; }
+
+/* Generic.Prompt */
+.highlight .gs {
+ font-weight: bold; }
+
+/* Generic.Strong */
+.highlight .gu {
+ color: #aaaaaa; }
+
+/* Generic.Subheading */
+.highlight .gt {
+ color: #aa0000; }
+
+/* Generic.Traceback */
+.highlight .kc {
+ font-weight: bold; }
+
+/* Keyword.Constant */
+.highlight .kd {
+ font-weight: bold; }
+
+/* Keyword.Declaration */
+.highlight .kp {
+ font-weight: bold; }
+
+/* Keyword.Pseudo */
+.highlight .kr {
+ font-weight: bold; }
+
+/* Keyword.Reserved */
+.highlight .kt {
+ color: #445588;
+ font-weight: bold; }
+
+/* Keyword.Type */
+.highlight .m {
+ color: #009999; }
+
+/* Literal.Number */
+.highlight .s {
+ color: #d14; }
+
+/* Literal.String */
+.highlight .na {
+ color: #008080; }
+
+/* Name.Attribute */
+.highlight .nb {
+ color: #0086B3; }
+
+/* Name.Builtin */
+.highlight .nc {
+ color: #445588;
+ font-weight: bold; }
+
+/* Name.Class */
+.highlight .no {
+ color: #008080; }
+
+/* Name.Constant */
+.highlight .ni {
+ color: #800080; }
+
+/* Name.Entity */
+.highlight .ne {
+ color: #990000;
+ font-weight: bold; }
+
+/* Name.Exception */
+.highlight .nf {
+ color: #990000;
+ font-weight: bold; }
+
+/* Name.Function */
+.highlight .nn {
+ color: #555555; }
+
+/* Name.Namespace */
+.highlight .nt {
+ color: #000080; }
+
+/* Name.Tag */
+.highlight .nv {
+ color: #008080; }
+
+/* Name.Variable */
+.highlight .ow {
+ font-weight: bold; }
+
+/* Operator.Word */
+.highlight .w {
+ color: #bbbbbb; }
+
+/* Text.Whitespace */
+.highlight .mf {
+ color: #009999; }
+
+/* Literal.Number.Float */
+.highlight .mh {
+ color: #009999; }
+
+/* Literal.Number.Hex */
+.highlight .mi {
+ color: #009999; }
+
+/* Literal.Number.Integer */
+.highlight .mo {
+ color: #009999; }
+
+/* Literal.Number.Oct */
+.highlight .sb {
+ color: #d14; }
+
+/* Literal.String.Backtick */
+.highlight .sc {
+ color: #d14; }
+
+/* Literal.String.Char */
+.highlight .sd {
+ color: #d14; }
+
+/* Literal.String.Doc */
+.highlight .s2 {
+ color: #d14; }
+
+/* Literal.String.Double */
+.highlight .se {
+ color: #d14; }
+
+/* Literal.String.Escape */
+.highlight .sh {
+ color: #d14; }
+
+/* Literal.String.Heredoc */
+.highlight .si {
+ color: #d14; }
+
+/* Literal.String.Interpol */
+.highlight .sx {
+ color: #d14; }
+
+/* Literal.String.Other */
+.highlight .sr {
+ color: #009926; }
+
+/* Literal.String.Regex */
+.highlight .s1 {
+ color: #d14; }
+
+/* Literal.String.Single */
+.highlight .ss {
+ color: #990073; }
+
+/* Literal.String.Symbol */
+.highlight .bp {
+ color: #999999; }
+
+/* Name.Builtin.Pseudo */
+.highlight .vc {
+ color: #008080; }
+
+/* Name.Variable.Class */
+.highlight .vg {
+ color: #008080; }
+
+/* Name.Variable.Global */
+.highlight .vi {
+ color: #008080; }
+
+/* Name.Variable.Instance */
+.highlight .il {
+ color: #009999; }
+
+/* Literal.Number.Integer.Long */
.main-nav {
float: right; }
.main-nav ul {
@@ -4673,6 +4927,9 @@ pre {
white-space: pre\9;
/* IE7+ */ }
+code {
+ overflow: scroll; }
+
.text-center {
text-align: center; }
@@ -4691,6 +4948,9 @@ hr {
ol {
padding: 0; }
+figure {
+ margin: 0; }
+
.ml-container {
display: -webkit-flex;
display: flex;
@@ -5436,4 +5696,4 @@ ol {
.preview-image img {
width: 500px; }
[... 3 lines stripped ...]
Modified: incubator/systemml/site/documentation.html
URL: http://svn.apache.org/viewvc/incubator/systemml/site/documentation.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/documentation.html (original)
+++ incubator/systemml/site/documentation.html Thu Apr 6 20:19:08 2017
@@ -117,9 +117,6 @@
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
-
- <li><a href="/security" target="_self">Security</a></li>
-
</ul>
</li>
Modified: incubator/systemml/site/download.html
URL: http://svn.apache.org/viewvc/incubator/systemml/site/download.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/download.html (original)
+++ incubator/systemml/site/download.html Thu Apr 6 20:19:08 2017
@@ -117,9 +117,6 @@
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
-
- <li><a href="/security" target="_self">Security</a></li>
-
</ul>
</li>
Modified: incubator/systemml/site/get-started.html
URL: http://svn.apache.org/viewvc/incubator/systemml/site/get-started.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/get-started.html (original)
+++ incubator/systemml/site/get-started.html Thu Apr 6 20:19:08 2017
@@ -181,9 +181,12 @@
<!-- Step 1 Code -->
<div class="col col-12">
- <pre><code>/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"
-# Linux
-ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Linuxbrew/install/master/install)"</code></pre>
+
+
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash">/usr/bin/ruby -e <span class="s2">"</span><span class="k">$(</span>curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install<span class="k">)</span><span class="s2">"</span>
+<span class="c"># Linux</span>
+ruby -e <span class="s2">"</span><span class="k">$(</span>curl -fsSL https://raw.githubusercontent.com/Linuxbrew/install/master/install<span class="k">)</span><span class="s2">"</span></code></pre></figure>
+
</div>
<!-- Step 2 Instructions -->
@@ -193,11 +196,11 @@ ruby -e "$(curl -fsSL https://raw.github
<!-- Step 2 Code -->
<div class="col col-12">
- <pre><code>brew tap caskroom/cask
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash">brew tap caskroom/cask
brew install Caskroom/cask/java
brew install python
-pip install jupyter matplotlib numpy</code></pre>
+pip install jupyter matplotlib numpy</code></pre></figure>
</div>
</div>
@@ -216,8 +219,8 @@ pip install jupyter matplotlib numpy</co
<!-- Step 3 Code -->
<div class="col col-12">
- <pre><code>brew tap homebrew/versions
-brew install apache-spark16</code></pre>
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash">brew tap homebrew/versions
+brew install apache-spark16</code></pre></figure>
<p> Alternatively, you can <a href="http://spark.apache.org/downloads.html">download Spark</a> directly. </p>
</div>
@@ -230,22 +233,24 @@ brew install apache-spark16</code></pre>
<!-- Step 4 Code -->
<div class="col col-12">
- <p>
- If you are a python user, we recommend that you download and install Apache SystemML via pip:
- </p>
- <pre><code>Python 2:
+ <p>
+ If you are a python user, we recommend that you download and install Apache SystemML via pip:
+ </p>
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="c"># Python 2</span>
pip install systemml
-# Bleeding edge: pip install git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python
+<span class="c"># Bleeding edge: pip install git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python</span></code></pre></figure>
-Python 3:
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash"><span class="c"># Python 3:</span>
pip3 install systemml
-# Bleeding edge: pip3 install git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python</code></pre>
+<span class="c"># Bleeding edge: pip3 install git+git://github.com/apache/incubator-systemml.git#subdirectory=src/main/python</span></code></pre></figure>
+
- <p>
- Alternatively, if you intend to use SystemML via spark-shell (or spark-submit), you only need systemml-0.12.0-incubating.jar, which is packaged into our official binary release (<a href="http://www.apache.org/dyn/closer.cgi/pub/apache/incubator/systemml/0.12.0-incubating/systemml-0.12.0-incubating.zip" target="_blank">systemml-0.12.0-incubating.zip</a>).
- Note: If you have installed SystemML via pip, you can get the location of this jar by executing following command:
- </p>
- <pre><code>python -c 'import imp; import os; print os.path.join(imp.find_module("systemml")[1], "systemml-java")'</code></pre>
+
+ <p>
+ Alternatively, if you intend to use SystemML via spark-shell (or spark-submit), you only need systemml-0.12.0-incubating.jar, which is packaged into our official binary release (<a href="http://www.apache.org/dyn/closer.cgi/pub/apache/incubator/systemml/0.12.0-incubating/systemml-0.12.0-incubating.zip" target="_blank">systemml-0.12.0-incubating.zip</a>).
+ Note: If you have installed SystemML via pip, you can get the location of this jar by executing following command:
+ </p>
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash">python -c <span class="s1">'import imp; import os; print os.path.join(imp.find_module("systemml")[1], "systemml-java")'</span></code></pre></figure>
</div>
@@ -254,31 +259,31 @@ pip3 install systemml
<!-- Section Header -->
<div class="col col-12 content-group--medium-bottom-margin">
<h2>Ways to Use</h2>
- <p>You can use SystemML in one of the following ways:
- <ol>
- <li>On Cluster (using our programmatic APIs):
- <ul>
- <li>Using pyspark: Please see our <a href="http://apache.github.io/incubator-systemml/beginners-guide-python">beginner's guide for python users</a>.</li>
- <li>Using Jupyter: Described below in step 5.</li>
- <li>Using spark-shell: Described below in step 6.</li>
- </ul>
- </li>
-
- <li>On Cluster (command-line batch mode):
- <ul>
- <li>Using spark-submit: Please see our <a href="http://apache.github.io/incubator-systemml/spark-batch-mode">spark batch mode tutorial</a>.</li>
- <li>Using hadoop: Please see our <a href="http://apache.github.io/incubator-systemml/hadoop-batch-mode">hadoop batch model tutorial</a>.</li>
- </ul>
- </li>
-
- <li>On laptop (command-line batch mode) without installing Spark or Hadoop: Please see our <a href="http://apache.github.io/incubator-systemml/standalone-guide">standalone mode tutorial</a>.</li>
-
- <li>In-memory mode (as part of another Java application for scoring): Please see our <a href="http://apache.github.io/incubator-systemml/jmlc">JMLC tutorial</a>.</li>
- </ol>
-
- <p>
- Note that you can also run pyspark, spark-shell, spark-submit on you laptop using "--master local[*]" parameter.
- </p>
+ <p>You can use SystemML in one of the following ways:</p>
+ <ol>
+ <li>On Cluster (using our programmatic APIs):
+ <ul>
+ <li>Using pyspark: Please see our <a href="http://apache.github.io/incubator-systemml/beginners-guide-python">beginner's guide for python users</a>.</li>
+ <li>Using Jupyter: Described below in step 5.</li>
+ <li>Using spark-shell: Described below in step 6.</li>
+ </ul>
+ </li>
+
+ <li>On Cluster (command-line batch mode):
+ <ul>
+ <li>Using spark-submit: Please see our <a href="http://apache.github.io/incubator-systemml/spark-batch-mode">spark batch mode tutorial</a>.</li>
+ <li>Using hadoop: Please see our <a href="http://apache.github.io/incubator-systemml/hadoop-batch-mode">hadoop batch model tutorial</a>.</li>
+ </ul>
+ </li>
+
+ <li>On laptop (command-line batch mode) without installing Spark or Hadoop: Please see our <a href="http://apache.github.io/incubator-systemml/standalone-guide">standalone mode tutorial</a>.</li>
+
+ <li>In-memory mode (as part of another Java application for scoring): Please see our <a href="http://apache.github.io/incubator-systemml/jmlc">JMLC tutorial</a>.</li>
+ </ol>
+
+ <p>
+ Note that you can also run pyspark, spark-shell, spark-submit on you laptop using "--master local[*]" parameter.
+ </p>
</div>
<!-- Step 5 Instructions -->
@@ -290,11 +295,10 @@ pip3 install systemml
<div class="col col-12">
<h4>Get Started</h4>
<p>Start up your Jupyter notebook by moving to the folder where you saved the notebook. Then copy and paste the line below:</p>
- <pre><code>Python 2:
-PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark --master local[*] --driver-class-path SystemML.jar --jars SystemML.jar--conf "spark.driver.memory=12g" --conf spark.driver.maxResultSize=0 --conf spark.akka.frameSize=128 --conf spark.default.parallelism=100
-
-Python 3:
-PYSPARK_PYTHON=python3 PYSPARK_DRIVER_PYTHON=jupyter PYSPARK_DRIVER_PYTHON_OPTS="notebook" pyspark --master local[*] --driver-class-path SystemML.jar --jars SystemML.jar --conf "spark.driver.memory=12g" --conf spark.driver.maxResultSize=0 --conf spark.akka.frameSize=128 --conf spark.default.parallelism=100</code></pre>
+ <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="c"># Python 2:</span>
+<span class="n">PYSPARK_DRIVER_PYTHON</span><span class="o">=</span><span class="n">jupyter</span> <span class="n">PYSPARK_DRIVER_PYTHON_OPTS</span><span class="o">=</span><span class="s">"notebook"</span> <span class="n">pyspark</span> <span class="o">--</span><span class="n">master</span> <span class="n">local</span><span class="p">[</span><span class="o">*</span><span class="p">]</span> <span class="o">--</span><span class="n">driver</span><span class="o">-</span><span class="n">class</span><span class="o">-</span><span class="n">path</span> <span class="n">SystemML</span><span class="o">.</span><span class="n">jar</span> <span class="o">--</span><span class="n">jars</span> <span class="n">SystemML</span><span class="o">.</span><span class="n">jar</span><span class="o">--</span><span class="n">conf</span> <span class="s">"spark.driver.memory=12g"</span> <span class="o">--</span><span class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span class="n">driver<
/span><span class="o">.</span><span class="n">maxResultSize</span><span class="o">=</span><span class="mi">0</span> <span class="o">--</span><span class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span class="n">akka</span><span class="o">.</span><span class="n">frameSize</span><span class="o">=</span><span class="mi">128</span> <span class="o">--</span><span class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span class="n">default</span><span class="o">.</span><span class="n">parallelism</span><span class="o">=</span><span class="mi">100</span></code></pre></figure>
+ <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="c"># Python 3:</span>
+<span class="n">PYSPARK_PYTHON</span><span class="o">=</span><span class="n">python3</span> <span class="n">PYSPARK_DRIVER_PYTHON</span><span class="o">=</span><span class="n">jupyter</span> <span class="n">PYSPARK_DRIVER_PYTHON_OPTS</span><span class="o">=</span><span class="s">"notebook"</span> <span class="n">pyspark</span> <span class="o">--</span><span class="n">master</span> <span class="n">local</span><span class="p">[</span><span class="o">*</span><span class="p">]</span> <span class="o">--</span><span class="n">driver</span><span class="o">-</span><span class="n">class</span><span class="o">-</span><span class="n">path</span> <span class="n">SystemML</span><span class="o">.</span><span class="n">jar</span> <span class="o">--</span><span class="n">jars</span> <span class="n">SystemML</span><span class="o">.</span><span class="n">jar</span> <span class="o">--</span><span class="n">conf</span> <span class="s">"spark.driver.memory=12g"</span> <span class="o">--</span><span clas
s="n">conf</span> <span class="n">spark</span><span class="o">.</span><span class="n">driver</span><span class="o">.</span><span class="n">maxResultSize</span><span class="o">=</span><span class="mi">0</span> <span class="o">--</span><span class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span class="n">akka</span><span class="o">.</span><span class="n">frameSize</span><span class="o">=</span><span class="mi">128</span> <span class="o">--</span><span class="n">conf</span> <span class="n">spark</span><span class="o">.</span><span class="n">default</span><span class="o">.</span><span class="n">parallelism</span><span class="o">=</span><span class="mi">100</span></code></pre></figure>
</div>
@@ -307,64 +311,67 @@ PYSPARK_PYTHON=python3 PYSPARK_DRIVER_PY
<div class="col col-12">
<h4>Start Spark Shell with SystemML</h4>
<p> To use SystemML with Spark Shell, the SystemML jar can be referenced using Spark Shell’s --jars option. Start the Spark Shell with SystemML with the following line of code in your terminal:</p>
- <pre><code>spark-shell --executor-memory 4G --driver-memory 4G --jars SystemML.jar</code></pre>
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash">spark-shell --executor-memory 4G --driver-memory 4G --jars SystemML.jar</code></pre></figure>
+ <!-- <pre><code>spark-shell --executor-memory 4G --driver-memory 4G --jars SystemML.jar</code></pre> -->
<h4>Create the MLContext</h4>
<p>To begin, start an MLContext by typing the code below. Once successful, you should see a “Welcome to Apache SystemML!” message.</p>
- <pre><code>import org.apache.sysml.api.mlcontext._
+ <figure class="highlight"><pre><code class="language-bash" data-lang="bash">import org.apache.sysml.api.mlcontext._
import org.apache.sysml.api.mlcontext.ScriptFactory._
-val ml = new MLContext(sc)</code></pre>
+val ml <span class="o">=</span> new MLContext<span class="o">(</span>sc<span class="o">)</span></code></pre></figure>
+
<h4>Hello World</h4>
<p>The ScriptFactory class allows DML and PYDML scripts to be created from Strings, Files, URLs, and InputStreams. Here, we’ll use the dmlmethod to create a DML “hello world” script based on a String. We execute the script using MLContext’s execute method, which displays “hello world” to the console. The execute method returns an MLResults object, which contains no results since the script has no outputs.</p>
- <pre><code>val helloScript = dml("print('hello world')")
-ml.execute(helloScript)</code></pre>
+ <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="n">val</span> <span class="n">helloScript</span> <span class="o">=</span> <span class="n">dml</span><span class="p">(</span><span class="s">"print('hello world')"</span><span class="p">)</span>
+<span class="n">ml</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">helloScript</span><span class="p">)</span></code></pre></figure>
<h4>DataFrame Example</h4>
<p>As an example of how to use SystemML, we’ll first use Spark to create a DataFrame called df of random doubles from 0 to 1 consisting of 10,000 rows and 1,000 columns.</p>
- <pre><code>import org.apache.spark.sql._
-import org.apache.spark.sql.types.{StructType,StructField,DoubleType}
-import scala.util.Random
-val numRows = 10000
-val numCols = 1000
-val data = sc.parallelize(0 to numRows-1).map { _ => Row.fromSeq(Seq.fill(numCols)(Random.nextDouble)) }
-val schema = StructType((0 to numCols-1).map { i => StructField("C" + i, DoubleType, true) } )
-val df = sqlContext.createDataFrame(data, schema)</code></pre>
+ <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="kn">import</span> <span class="nn">org.apache.spark.sql._</span>
+<span class="kn">import</span> <span class="nn">org.apache.spark.sql.types.</span><span class="p">{</span><span class="n">StructType</span><span class="p">,</span><span class="n">StructField</span><span class="p">,</span><span class="n">DoubleType</span><span class="p">}</span>
+<span class="kn">import</span> <span class="nn">scala.util.Random</span>
+<span class="n">val</span> <span class="n">numRows</span> <span class="o">=</span> <span class="mi">10000</span>
+<span class="n">val</span> <span class="n">numCols</span> <span class="o">=</span> <span class="mi">1000</span>
+<span class="n">val</span> <span class="n">data</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="mi">0</span> <span class="n">to</span> <span class="n">numRows</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="nb">map</span> <span class="p">{</span> <span class="n">_</span> <span class="o">=></span> <span class="n">Row</span><span class="o">.</span><span class="n">fromSeq</span><span class="p">(</span><span class="n">Seq</span><span class="o">.</span><span class="n">fill</span><span class="p">(</span><span class="n">numCols</span><span class="p">)(</span><span class="n">Random</span><span class="o">.</span><span class="n">nextDouble</span><span class="p">))</span> <span class="p">}</span>
+<span class="n">val</span> <span class="n">schema</span> <span class="o">=</span> <span class="n">StructType</span><span class="p">((</span><span class="mi">0</span> <span class="n">to</span> <span class="n">numCols</span><span class="o">-</span><span class="mi">1</span><span class="p">)</span><span class="o">.</span><span class="nb">map</span> <span class="p">{</span> <span class="n">i</span> <span class="o">=></span> <span class="n">StructField</span><span class="p">(</span><span class="s">"C"</span> <span class="o">+</span> <span class="n">i</span><span class="p">,</span> <span class="n">DoubleType</span><span class="p">,</span> <span class="n">true</span><span class="p">)</span> <span class="p">}</span> <span class="p">)</span>
+<span class="n">val</span> <span class="n">df</span> <span class="o">=</span> <span class="n">sqlContext</span><span class="o">.</span><span class="n">createDataFrame</span><span class="p">(</span><span class="n">data</span><span class="p">,</span> <span class="n">schema</span><span class="p">)</span></code></pre></figure>
+
<p>We’ll create a DML script using the ScriptFactory dml method to find the minimum, maximum, and mean values in a matrix. This script has one input variable, matrix Xin, and three output variables, minOut, maxOut, and meanOut.
For performance, we’ll specify metadata indicating that the matrix has 10,000 rows and 1,000 columns.
We execute the script and obtain the results as a Tuple by calling getTuple on the results, specifying the types and names of the output variables.</p>
- <pre><code>val minMaxMean =
-"""
+ <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="n">val</span> <span class="n">minMaxMean</span> <span class="o">=</span>
+<span class="s">"""
minOut = min(Xin)
maxOut = max(Xin)
meanOut = mean(Xin)
-"""
-val mm = new MatrixMetadata(numRows, numCols)
-val minMaxMeanScript = dml(minMaxMean).in("Xin", df, mm).out("minOut", "maxOut", "meanOut")
-val (min, max, mean) = ml.execute(minMaxMeanScript).getTuple[Double, Double, Double]("minOut", "maxOut", "meanOut")</code></pre>
+"""</span>
+<span class="n">val</span> <span class="n">mm</span> <span class="o">=</span> <span class="n">new</span> <span class="n">MatrixMetadata</span><span class="p">(</span><span class="n">numRows</span><span class="p">,</span> <span class="n">numCols</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">minMaxMeanScript</span> <span class="o">=</span> <span class="n">dml</span><span class="p">(</span><span class="n">minMaxMean</span><span class="p">)</span><span class="o">.</span><span class="ow">in</span><span class="p">(</span><span class="s">"Xin"</span><span class="p">,</span> <span class="n">df</span><span class="p">,</span> <span class="n">mm</span><span class="p">)</span><span class="o">.</span><span class="n">out</span><span class="p">(</span><span class="s">"minOut"</span><span class="p">,</span> <span class="s">"maxOut"</span><span class="p">,</span> <span class="s">"meanOut"</span><span class="p">)</span>
+<span class="n">val</span> <span class="p">(</span><span class="nb">min</span><span class="p">,</span> <span class="nb">max</span><span class="p">,</span> <span class="n">mean</span><span class="p">)</span> <span class="o">=</span> <span class="n">ml</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">minMaxMeanScript</span><span class="p">)</span><span class="o">.</span><span class="n">getTuple</span><span class="p">[</span><span class="n">Double</span><span class="p">,</span> <span class="n">Double</span><span class="p">,</span> <span class="n">Double</span><span class="p">](</span><span class="s">"minOut"</span><span class="p">,</span> <span class="s">"maxOut"</span><span class="p">,</span> <span class="s">"meanOut"</span><span class="p">)</span></code></pre></figure>
<p>Many different types of input and output variables are automatically allowed. These types include Boolean, Long, Double, String, Array[Array[Double]], RDD<String> and JavaRDD<String> in CSV (dense) and IJV (sparse) formats, DataFrame, BinaryBlockMatrix,Matrix, and Frame. RDDs and JavaRDDs are assumed to be CSV format unless MatrixMetadata is supplied indicating IJV format.</p>
<h4>RDD Example:</h4>
<p>Let’s take a look at an example of input matrices as RDDs in CSV format. We’ll create two 2x2 matrices and input these into a DML script. This script will sum each matrix and create a message based on which sum is greater. We will output the sums and the message.</p>
-<pre><code>val rdd1 = sc.parallelize(Array("1.0,2.0", "3.0,4.0"))
-val rdd2 = sc.parallelize(Array("5.0,6.0", "7.0,8.0"))
-val sums = """
+ <figure class="highlight"><pre><code class="language-python" data-lang="python"><span class="n">val</span> <span class="n">rdd1</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">Array</span><span class="p">(</span><span class="s">"1.0,2.0"</span><span class="p">,</span> <span class="s">"3.0,4.0"</span><span class="p">))</span>
+<span class="n">val</span> <span class="n">rdd2</span> <span class="o">=</span> <span class="n">sc</span><span class="o">.</span><span class="n">parallelize</span><span class="p">(</span><span class="n">Array</span><span class="p">(</span><span class="s">"5.0,6.0"</span><span class="p">,</span> <span class="s">"7.0,8.0"</span><span class="p">))</span>
+<span class="n">val</span> <span class="n">sums</span> <span class="o">=</span> <span class="s">"""
s1 = sum(m1);
s2 = sum(m2);
-if (s1 > s2) {
+if (s1 > s2) {
message = "s1 is greater"
-} else if (s2 > s1) {
+} else if (s2 > s1) {
message = "s2 is greater"
} else {
message = "s1 and s2 are equal"
}
-"""
-scala.tools.nsc.io.File("sums.dml").writeAll(sums)
-val sumScript = dmlFromFile("sums.dml").in(Map("m1"-> rdd1, "m2"-> rdd2)).out("s1", "s2", "message")
-val sumResults = ml.execute(sumScript)
-val s1 = sumResults.getDouble("s1")
-val s2 = sumResults.getDouble("s2")
-val message = sumResults.getString("message")
-val rdd1Metadata = new MatrixMetadata(2, 2)
-val rdd2Metadata = new MatrixMetadata(2, 2)
-val sumScript = dmlFromFile("sums.dml").in(Seq(("m1", rdd1, rdd1Metadata), ("m2", rdd2, rdd2Metadata))).out("s1", "s2", "message")
-val (firstSum, secondSum, sumMessage) = ml.execute(sumScript).getTuple[Double, Double, String]("s1", "s2", "message")</code></pre>
+"""</span>
+<span class="n">scala</span><span class="o">.</span><span class="n">tools</span><span class="o">.</span><span class="n">nsc</span><span class="o">.</span><span class="n">io</span><span class="o">.</span><span class="n">File</span><span class="p">(</span><span class="s">"sums.dml"</span><span class="p">)</span><span class="o">.</span><span class="n">writeAll</span><span class="p">(</span><span class="n">sums</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">sumScript</span> <span class="o">=</span> <span class="n">dmlFromFile</span><span class="p">(</span><span class="s">"sums.dml"</span><span class="p">)</span><span class="o">.</span><span class="ow">in</span><span class="p">(</span><span class="n">Map</span><span class="p">(</span><span class="s">"m1"</span><span class="o">-></span> <span class="n">rdd1</span><span class="p">,</span> <span class="s">"m2"</span><span class="o">-></span> <span class="n">rdd2</span><span class="p">))</span><span class="o">.</span><span class="n">out</span><span class="p">(</span><span class="s">"s1"</span><span class="p">,</span> <span class="s">"s2"</span><span class="p">,</span> <span class="s">"message"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">sumResults</span> <span class="o">=</span> <span class="n">ml</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sumScript</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">s1</span> <span class="o">=</span> <span class="n">sumResults</span><span class="o">.</span><span class="n">getDouble</span><span class="p">(</span><span class="s">"s1"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">s2</span> <span class="o">=</span> <span class="n">sumResults</span><span class="o">.</span><span class="n">getDouble</span><span class="p">(</span><span class="s">"s2"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">message</span> <span class="o">=</span> <span class="n">sumResults</span><span class="o">.</span><span class="n">getString</span><span class="p">(</span><span class="s">"message"</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">rdd1Metadata</span> <span class="o">=</span> <span class="n">new</span> <span class="n">MatrixMetadata</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">rdd2Metadata</span> <span class="o">=</span> <span class="n">new</span> <span class="n">MatrixMetadata</span><span class="p">(</span><span class="mi">2</span><span class="p">,</span> <span class="mi">2</span><span class="p">)</span>
+<span class="n">val</span> <span class="n">sumScript</span> <span class="o">=</span> <span class="n">dmlFromFile</span><span class="p">(</span><span class="s">"sums.dml"</span><span class="p">)</span><span class="o">.</span><span class="ow">in</span><span class="p">(</span><span class="n">Seq</span><span class="p">((</span><span class="s">"m1"</span><span class="p">,</span> <span class="n">rdd1</span><span class="p">,</span> <span class="n">rdd1Metadata</span><span class="p">),</span> <span class="p">(</span><span class="s">"m2"</span><span class="p">,</span> <span class="n">rdd2</span><span class="p">,</span> <span class="n">rdd2Metadata</span><span class="p">)))</span><span class="o">.</span><span class="n">out</span><span class="p">(</span><span class="s">"s1"</span><span class="p">,</span> <span class="s">"s2"</span><span class="p">,</span> <span class="s">"message"</span><span class="p">)</span>
+<span class="n">val</span> <span class="p">(</span><span class="n">firstSum</span><span class="p">,</span> <span class="n">secondSum</span><span class="p">,</span> <span class="n">sumMessage</span><span class="p">)</span> <span class="o">=</span> <span class="n">ml</span><span class="o">.</span><span class="n">execute</span><span class="p">(</span><span class="n">sumScript</span><span class="p">)</span><span class="o">.</span><span class="n">getTuple</span><span class="p">[</span><span class="n">Double</span><span class="p">,</span> <span class="n">Double</span><span class="p">,</span> <span class="n">String</span><span class="p">](</span><span class="s">"s1"</span><span class="p">,</span> <span class="s">"s2"</span><span class="p">,</span> <span class="s">"message"</span><span class="p">)</span></code></pre></figure>
<p>Congratulations! You’ve now run examples in Apache SystemML!</p>
</div>
</div>
Modified: incubator/systemml/site/privacy-policy.html
URL: http://svn.apache.org/viewvc/incubator/systemml/site/privacy-policy.html?rev=1790452&r1=1790451&r2=1790452&view=diff
==============================================================================
--- incubator/systemml/site/privacy-policy.html (original)
+++ incubator/systemml/site/privacy-policy.html Thu Apr 6 20:19:08 2017
@@ -117,9 +117,6 @@
<li><a href="/privacy-policy" target="_self">Privacy Policy</a></li>
-
- <li><a href="/security" target="_self">Security</a></li>
-
</ul>
</li>