You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/01/22 21:33:26 UTC
svn commit: r1560502 [8/18] - in /incubator/spark: ./ _layouts/ _plugins/
css/ images/ js/ mllib/ news/ news/_posts/ releases/_posts/
screencasts/_posts/ site/ site/css/ site/images/ site/js/ site/mllib/
site/news/ site/releases/ site/screencasts/ site...
Added: incubator/spark/site/css/bootstrap.min.css
URL: http://svn.apache.org/viewvc/incubator/spark/site/css/bootstrap.min.css?rev=1560502&view=auto
==============================================================================
--- incubator/spark/site/css/bootstrap.min.css (added)
+++ incubator/spark/site/css/bootstrap.min.css Wed Jan 22 20:33:24 2014
@@ -0,0 +1,7 @@
+/*!
+ * Bootstrap v3.0.3 (http://getbootstrap.com)
+ * Copyright 2013 Twitter, Inc.
+ * Licensed under http://www.apache.org/licenses/LICENSE-2.0
+ */
+
[... 3 lines stripped ...]
Added: incubator/spark/site/css/cerulean.min.css
URL: http://svn.apache.org/viewvc/incubator/spark/site/css/cerulean.min.css?rev=1560502&view=auto
==============================================================================
--- incubator/spark/site/css/cerulean.min.css (added)
+++ incubator/spark/site/css/cerulean.min.css Wed Jan 22 20:33:24 2014
@@ -0,0 +1 @@
[... 3 lines stripped ...]
Added: incubator/spark/site/css/custom.css
URL: http://svn.apache.org/viewvc/incubator/spark/site/css/custom.css?rev=1560502&view=auto
==============================================================================
--- incubator/spark/site/css/custom.css (added)
+++ incubator/spark/site/css/custom.css Wed Jan 22 20:33:24 2014
@@ -0,0 +1,267 @@
+.tagline {
+ display: inline-block;
+ color: rgb(47, 164, 231);
+ padding-bottom: 5px;
+ font-style: italic;
+}
+
+.tagline {
+ font-size: 18px;
+}
+
+@media (min-width: 768px) {
+ .tagline {
+ font-size: 19px;
+ }
+}
+
+.row-padded {
+ margin-top: 30px;
+}
+
+.col-padded-top {
+ margin-top: 30px;
+ margin-bottom: 15px;
+}
+
+.code {
+ font-family: "Menlo", "Lucida Console", Consolas, monospace;
+ font-size: 12px;
+}
+
+@media (min-width: 1200px) {
+ .code {
+ font-size: 13px;
+ }
+}
+
+.code.code-big {
+ font-size: 14px;
+}
+
+@media (min-width: 992px) {
+ .code.code-big {
+ font-size: 16px;
+ }
+}
+
+.code .string {
+ color: #2b8eeb;
+}
+
+.code .sparkop {
+ color: #1663a8;
+}
+
+.code .closure {
+ color: #c1130e;
+}
+
+.code .comment {
+ color: #272;
+}
+
+.code .tgf {
+ color: #f28424;
+}
+
+.code .keyword {
+ font-weight: 600;
+}
+
+.caption {
+ width: 100%;
+ margin-top: 20px;
+ text-align: center;
+ color: #8f8f8f;
+}
+
+ul.nav li.dropdown:hover > ul.dropdown-menu {
+ display: block;
+}
+
+footer {
+ text-align: center;
+ color: #8f8f8f;
+ padding-bottom: 10px;
+}
+
+.jumbotron {
+ padding: 20px 20px;
+ margin-bottom: 20px;
+ color: rgb(85,85,85);
+ background: #eef6fd;
+ text-align: center;
+ line-height: inherit;
+}
+
+.jumbotron {
+ font-size: 18px;
+}
+
+@media (min-width: 768px) {
+ .jumbotron {
+ font-size: 19px;
+ }
+}
+
+@media (min-width: 992px) {
+ .jumbotron {
+ font-size: 21px;
+ }
+}
+
+.navbar {
+ background: #2fa4e7;
+ border: none;
+ box-shadow: none;
+ font-size: 14px;
+ font-weight: bold;
+}
+
+/* Lower height of navbar */
+
+.navbar-nav > li > a {
+ padding-top:12px !important;
+ padding-bottom:12px !important;
+}
+
+.navbar {
+ min-height:44px !important
+}
+
+.news {
+ background: #fef4e9;
+ border-radius: 6px;
+ padding: 4px 12px 1px 12px;
+}
+
+.news h1, .news h2, .news h3, .news h4, .news h5 {
+ color: rgb(85, 85, 85);
+}
+
+.news ul li {
+ margin-bottom: 6px;
+}
+
+.news li a, .news li a:hover, .news li a:visited {
+ color: rgb(85, 85, 85);
+}
+
+.news li .small {
+ /*display: inline-block;*/
+ color: #888;
+ font-size: 12px;
+}
+
+.news h5 {
+ font-size: 16px;
+}
+
+.col-right {
+ text-align: center;
+}
+
+.col-left {
+ text-align: center;
+}
+
+@media (min-width: 992px) {
+ .col-right {
+ text-align: right;
+ }
+}
+
+@media (min-width: 992px) {
+ .col-left {
+ text-align: left;
+ }
+}
+
+.col-center {
+ text-align: center;
+}
+
+.list-narrow {
+ padding-left: 1.5em;
+}
+
+.col-padded {
+ margin-top: 30px;
+ margin-bottom: 30px;
+}
+
+.subproject {
+ display: inline-block;
+ font-weight: 200;
+ color: #333;
+}
+
+.subproject {
+ font-size: 30px;
+}
+
+@media (min-width: 768px) {
+ .subproject {
+ font-size: 45px;
+ }
+}
+
+.btn-lg .small {
+ font-size: 14px;
+}
+
+.navbar-collapse.in {
+ overflow-y: visible;
+ max-height: none;
+}
+
+.question {
+ font-size: 16px;
+ margin-top: 20px;
+ color: #555;
+ font-weight: 500;
+}
+
+.entry-date {
+ color: #888;
+ font-size: 14px;
+}
+
+.entry-header {
+ margin-top: 30px;
+ margin-bottom: 10px;
+}
+
+code {
+ font-family: "Menlo", "Lucida Console", Consolas, monospace;
+ background: transparent;
+ padding: 0;
+ color: inherit;
+}
+
+/* Designed to be shown under some tabs */
+.code-tab {
+ margin-bottom: 10px;
+ padding: 10px 0px;
+ border-bottom: 1px solid #ddd;
+}
+
+.btn-multiline {
+ width: 280px;
+ padding-top: 11px;
+ padding-bottom: 11px;
+ line-height: 1.25;
+}
+
+.nav.nav-tabs>li>a {
+ padding: 6px 11px;
+ font-size: 14px;
+ border-radius: 0;
+}
+
+.tabs-below > .nav-tabs,
+.tabs-right > .nav-tabs,
+.tabs-left > .nav-tabs {
+ border-bottom: 0;
+}
Modified: incubator/spark/site/documentation.html
URL: http://svn.apache.org/viewvc/incubator/spark/site/documentation.html?rev=1560502&r1=1560501&r2=1560502&view=diff
==============================================================================
--- incubator/spark/site/documentation.html (original)
+++ incubator/spark/site/documentation.html Wed Jan 22 20:33:24 2014
@@ -1,27 +1,20 @@
<!DOCTYPE html>
-<!--[if IE 6]>
-<html id="ie6" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 7]>
-<html id="ie7" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 8]>
-<html id="ie8" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if !(IE 6) | !(IE 7) | !(IE 8) ]><!-->
-<html dir="ltr" lang="en-US">
-<!--<![endif]-->
+<html lang="en">
<head>
- <link rel="shortcut icon" href="/favicon.ico" />
- <meta charset="UTF-8" />
- <meta name="viewport" content="width=device-width" />
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
<title>
Documentation | Apache Spark
</title>
- <link rel="stylesheet" type="text/css" media="all" href="/css/style.css" />
- <link rel="stylesheet" href="/css/pygments-default.css">
+
+
+ <!-- Bootstrap core CSS -->
+ <link href="/css/cerulean.min.css" rel="stylesheet">
+ <link href="/css/custom.css" rel="stylesheet">
<script type="text/javascript">
<!-- Google Analytics initialization -->
@@ -46,102 +39,137 @@
}
</script>
- <link rel='canonical' href='/index.html' />
-
- <style type="text/css">
- #site-title,
- #site-description {
- position: absolute !important;
- clip: rect(1px 1px 1px 1px); /* IE6, IE7 */
- clip: rect(1px, 1px, 1px, 1px);
- }
- </style>
- <style type="text/css" id="custom-background-css">
- body.custom-background { background-color: #f1f1f1; }
- </style>
+ <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
+ <![endif]-->
</head>
-<!--body class="page singular"-->
-<body class="page singular">
-<div id="page" class="hfeed">
-
- <header id="branding" role="banner">
- <hgroup>
- <h1 id="site-title"><span><a href="/" title="Spark" rel="home">Spark</a></span></h1>
- <h2 id="site-description">Lightning-Fast Cluster Computing</h2>
- </hgroup>
-
- <a id="main-logo" href="/">
- <img style="height:175px; width:auto;" src="/images/spark-project-header1-cropped.png" alt="Spark: Lightning-Fast Cluster Computing" title="Spark: Lightning-Fast Cluster Computing" />
- </a>
- <div class="widget-summit">
- <a href="http://spark-summit.org"><img src="/images/Summit-Logo-FINALtr-150x150px.png" /></a>
- <div class="text">
- <a href="http://spark-summit.org/2013">
-
- <strong>Videos and Slides<br/>
- Available Now!</strong>
- </a>
- </div>
+<body>
+
+<div class="container" style="max-width: 1200px;">
+
+<div class="masthead">
+
+ <p class="lead">
+ <a href="/">
+ <img src="/images/spark-logo.png"
+ style="height:100px; width:auto; vertical-align: bottom; margin-top: 20px;"></a><span class="tagline">
+ Lightning-fast cluster computing
+ </span>
+ </p>
+
+</div>
+
+<nav class="navbar navbar-default" role="navigation">
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse"
+ data-target="#navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
</div>
- <nav id="access" role="navigation">
- <h3 class="assistive-text">Main menu</h3>
- <div class="menu-main-menu-container">
- <ul id="menu-main-menu" class="menu">
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/index.html">Home</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/downloads.html">Downloads</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page current-menu-item">
- <a href="/documentation.html">Documentation</a>
- </li>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/downloads.html">Download</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Related Projects <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Documentation <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/documentation.html">Overview</a></li>
+ <li><a href="/docs/latest/">Latest Release</a></li>
+ <li><a href="/examples.html">Examples</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Community <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/community.html">Mailing Lists</a></li>
+ <li><a href="/community.html#events">Events and Meetups</a></li>
+ <li><a href="/community.html#history">Project History</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SPARK/Powered+By+Spark">Powered By</a></li>
+ </ul>
+ </li>
+ <li><a href="/faq.html">FAQ</a></li>
+ </ul>
+ </div>
+ <!-- /.navbar-collapse -->
+</nav>
+
+
+<div class="row">
+ <div class="col-md-3 col-md-push-9">
+ <div class="news" style="margin-bottom: 20px;">
+ <h5>Latest News</h5>
+ <ul class="list-unstyled">
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/examples.html">Examples</a>
- </li>
+ <li><a href="/news/spark-0-8-1-released.html">Spark 0.8.1 released</a>
+ <span class="small">(Dec 19, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/mailing-lists.html">Mailing Lists</a>
- </li>
+ <li><a href="/news/spark-summit-2013-is-a-wrap.html">Spark Summit 2013 is a Wrap</a>
+ <span class="small">(Dec 15, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/research.html">Research</a>
- </li>
+ <li><a href="/news/announcing-the-first-spark-summit.html">Announcing the first Spark Summit: December 2, 2013</a>
+ <span class="small">(Oct 08, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/faq.html">FAQ</a>
- </li>
+ <li><a href="/news/spark-0-8-0-released.html">Spark 0.8.0 released</a>
+ <span class="small">(Sep 25, 2013)</span></li>
- </ul></div>
- </nav><!-- #access -->
-</header><!-- #branding -->
-
-
+ </ul>
+ <p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
+ </div>
+ <div class="hidden-xs hidden-sm">
+ <a href="/downloads.html" class="btn btn-success btn-lg btn-block" style="margin-bottom: 30px;">
+ Download Spark
+ </a>
+ <p style="font-size: 16px; font-weight: 500; color: #555;">
+ Related Projects:
+ </p>
+ <ul class="list-narrow">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </div>
+ </div>
- <div id="main">
- <div id="primary">
- <div id="content" role="main">
-
- <article class="page type-page status-publish hentry">
- <h2>Spark Documentation</h2>
+ <div class="col-md-9 col-md-pull-3">
+ <h2>Spark Documentation</h2>
<p>Setup instructions, programming guides, and other documentation are available for each version of Spark below:</p>
<ul>
<li><a href="/docs/latest/">Spark 0.8.1 (latest release)</a></li>
- <li><a href="/docs/0.8.0/">Spark 0.8.0</a></li>
<li><a href="/docs/0.7.3/">Spark 0.7.3</a></li>
<li><a href="/docs/0.6.2/">Spark 0.6.2</a></li>
- <li><a href="https://github.com/mesos/spark/wiki/Spark-0.5-Documentation">Spark 0.5.x</a> (hosted on GitHub)</li>
</ul>
-<p>Read these documents to get started with Spark. In addition, this page lists some external resources for learning Spark.</p>
+<p>Read these documents to get started with Spark, as well as with the built-in components
+(<a href="/docs/latest/mllib-guide.html">MLlib</a> and
+<a href="/docs/latest/streaming-programming-guide.html">Spark Streaming</a>).</p>
+
+<p>In addition, this page lists some external resources for learning Spark.</p>
<h3>Video Tutorials</h3>
@@ -156,23 +184,37 @@
<h3>Hands-On Exercises</h3>
<ul>
- <li><a href="http://ampcamp.berkeley.edu/3/exercises/">Hands-on exercises</a> are available online. These exercises let you launch a small EC2 cluster, load a dataset, and query it with Spark, Shark, Spark Streaming, and MLLib.</li>
+ <li><a href="http://spark-summit.org/2013/exercises/">Hands-on exercises</a> are available online from Spark Summit 2013. These exercises let you launch a small EC2 cluster, load a dataset, and query it with Spark, Shark, Spark Streaming, and MLLib.</li>
</ul>
-<h3>Spark Summit Slides and Videos</h3>
+<p><a name="summit"></a></p>
+<h3>Training Materials</h3>
+<ul>
+ <li><a href="http://spark-summit.org/2013">Spark Summit 2013</a> contained a training session for which
+ <a href="http://spark-summit.org/summit-2013/#day2">slides and videos</a> are available for free online.
+ The session also included <a href="http://spark-summit.org/2013/exercises/">exercises</a> that you can run on Amazon EC2.</li>
+ <li>The <a href="https://amplab.cs.berkeley.edu/">UC Berkeley AMPLab</a> regularly hosts training camps on Spark and related projects.
+Slides and videos are available online:
<ul>
- <li><a href="http://spark-summit.org/2013">Spark Summit 2013</a> was held in downtown San Francisco in December 2013. Slides and Videos of all talks are available for free. Look for links next to talk titles on the event agenda.</li>
+ <li><a href="http://ampcamp.berkeley.edu/3/">AMP Camp Three</a> (Berkeley, CA, August 2013)</li>
+ <li><a href="http://ampcamp.berkeley.edu/amp-camp-two-strata-2013/">AMP Camp Two</a> (Strata Santa Clara, February 2013)</li>
+ <li><a href="http://ampcamp.berkeley.edu/agenda-2012/">AMP Camp One</a> (Berkeley, CA, August 2012)</li>
+ </ul>
+ </li>
</ul>
-<h3>AMP Camp Slides and Videos</h3>
+<h3>External Tutorials, Blog Posts, and Talks</h3>
<ul>
- <li>The <a href="https://amplab.cs.berkeley.edu/">UC Berkeley AMPLab</a> regularly hosts two-day training camps on Spark and related "big data" components.
-Slides and videos from each camp are posted online:
- <br /><a href="http://ampcamp.berkeley.edu/3/">AMP Camp Three</a> <em>Big Data Bootcamp Berkeley</em> (August 2013)
- <br /><a href="http://ampcamp.berkeley.edu/amp-camp-two-strata-2013/">AMP Camp Two</a> <em>Big Data Bootcamp Strata</em> (February 2013)
- <br /><a href="http://ampcamp.berkeley.edu/agenda-2012/">AMP Camp One</a> <em>Big Data Bootcamp Berkeley</em> (August 2012)
- </li>
+ <li><a href="http://spark-summit.org/2013">Spark Summit 2013</a> — contained 30 talks about Spark use cases, available as slides and videos</li>
+ <li><a href="http://www.pwendell.com/2013/09/28/declarative-streams.html">Sampling Twitter Using Declarative Streams</a> — Spark Streaming tutorial by Patrick Wendell</li>
+ <li><a href="http://zenfractal.com/2013/08/21/a-powerful-big-data-trio/">A Powerful Big Data Trio: Spark, Parquet and Avro</a> — Using Parquet in Spark by Matt Massie</li>
+ <li><a href="http://www.slideshare.net/EvanChan2/cassandra2013-spark-talk-final">Real-time Analytics with Cassandra, Spark, and Shark</a> — Presentation by Evan Chan from Ooyala at 2013 Cassandra Summit</li>
+ <li><a href="http://syndeticlogic.net/?p=311">Getting Spark Setup in Eclipse</a> — Developer blog post by James Percent</li>
+ <li><a href="http://aws.amazon.com/articles/Elastic-MapReduce/4926593393724923">Run Spark and Shark on Amazon Elastic MapReduce</a> — Article by Amazon Elastic MapReduce team member Parviz Deyhim</li>
+ <li><a href="http://blog.quantifind.com/posts/spark-unit-test/">Unit testing with Spark</a> — Quantifind tech blog post by Imran Rashid</li>
+ <li><a href="http://blog.quantifind.com/posts/logging-post/">Configuring Spark logs</a> — Quantifind tech blog by Imran Rashid</li>
+ <li><a href="http://www.ibm.com/developerworks/library/os-spark/">Spark, an alternative for fast data analytics</a> — IBM Developer Works article by M. Tim Jones</li>
</ul>
<h3>Books</h3>
@@ -181,27 +223,27 @@ Slides and videos from each camp are pos
<li><a href="http://www.packtpub.com/fast-data-processing-with-spark/book">Fast Data Processing with Spark</a>, by Holden Karau (Packt Publishing)</li>
</ul>
-<h3>External Tutorials, Development Blogs, and Talks</h3>
+<h3>Examples</h3>
<ul>
- <li><a href="http://www.pwendell.com/2013/09/28/declarative-streams.html">Sampling Twitter Using Declarative Streams</a> -- Spark Streaming tutorial by Patrick Wendell</li>
- <li><a href="http://zenfractal.com/2013/08/21/a-powerful-big-data-trio/">A Powerful Big Data Trio: Spark, Parquet and Avro</a> -- Using Parquet in Spark by Matt Massie</li>
- <li><a href="http://www.slideshare.net/EvanChan2/cassandra2013-spark-talk-final">Real-time Analytics with Cassandra, Spark, and Shark</a> -- Presentation by Evan Chan from Ooyala at the 2013 Cassandra Summit</li>
- <li><a href="http://syndeticlogic.net/?p=311">Getting Spark Setup in Eclipse</a> -- Developer blog post by James Percent</li>
- <li><a href="http://aws.amazon.com/articles/Elastic-MapReduce/4926593393724923">Run Spark and Shark on Amazon Elastic MapReduce</a> -- Article by Amazon AWS Elastic MapReduce team member Parviz Deyhim</li>
- <li><a href="http://blog.quantifind.com/posts/spark-unit-test/">Unit testing with Spark</a> -- Quantifind tech blog post by Imran Rashid</li>
- <li><a href="http://blog.quantifind.com/posts/logging-post/">Configuring Spark logs</a> -- Quantifind tech blog by Imran Rashid</li>
- <li><a href="http://www.ibm.com/developerworks/library/os-spark/">Spark, an alternative for fast data analytics</a> -- IBM Developer Works article by M. Tim Jones</li>
+ <li>The <a href="/examples.html">Spark examples page</a> shows the basic API in Scala, Java and Python.</li>
</ul>
-<h3>Spark Internals</h3>
+<h3>Wiki</h3>
-<ul>
- <li><a href="http://www.youtube.com/watch?v=49Hr5xZyTEA">Overview of Spark Internals [advanced]</a> (<a href="/talks/dev-meetup-dec-2012.pptx">pptx</a>) (<a href="http://www.youtube.com/watch?v=49Hr5xZyTEA">video</a>)</li>
-</ul>
+<ul><li>
+The <a href="https://cwiki.apache.org/confluence/display/SPARK/Wiki+Homepage">Spark wiki</a> contains
+information for developers, such as architecture documents and how to <a href="https://cwiki.apache.org/confluence/display/SPARK/Contributing+to+Spark">contribute</a> to Spark.
+</li></ul>
<h3>Research Papers</h3>
+<p>
+Spark was initially developed as a UC Berkeley research project, and much of the design is documented in papers.
+The <a href="/research.html">research page</a> lists some of the original motivation and direction.
+The following papers have been published about Spark and related projects.
+</p>
+
<ul>
<li>
<a href="http://www.eecs.berkeley.edu/Pubs/TechRpts/2012/EECS-2012-214.pdf">Shark: SQL and Rich Analytics at Scale</a>. Reynold Xin, Joshua Rosen, Matei Zaharia, Michael J. Franklin, Scott Shenker, Ion Stoica. <em>Technical Report UCB/EECS-2012-214</em>. November 2012.
@@ -222,25 +264,25 @@ Slides and videos from each camp are pos
</li>
</ul>
- </article><!-- #post -->
-
- </div><!-- #content -->
-
- <footer id="colophon" role="contentinfo">
- <div id="site-generator">
- <p style="padding-top: 0; padding-bottom: 15px;">
- Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
- <a href="http://incubator.apache.org/" style="border: none;">
- <img style="vertical-align: middle; border: none;" src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
- </a>
- </p>
</div>
-</footer><!-- #colophon -->
+</div>
+
+
+
+<footer class="small">
+ <hr>
+ Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
+ <a href="http://incubator.apache.org/" style="border: none;">
+ <img style="vertical-align: middle; float: right; margin-bottom: 15px;"
+ src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
+ </a>
+</footer>
- </div><!-- #primary -->
- </div><!-- #main -->
-</div><!-- #page -->
+</div>
+<script src="https://code.jquery.com/jquery.js"></script>
+<script src="//netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js"></script>
+<script src="/js/lang-tabs.js"></script>
</body>
</html>
Modified: incubator/spark/site/downloads.html
URL: http://svn.apache.org/viewvc/incubator/spark/site/downloads.html?rev=1560502&r1=1560501&r2=1560502&view=diff
==============================================================================
--- incubator/spark/site/downloads.html (original)
+++ incubator/spark/site/downloads.html Wed Jan 22 20:33:24 2014
@@ -1,27 +1,20 @@
<!DOCTYPE html>
-<!--[if IE 6]>
-<html id="ie6" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 7]>
-<html id="ie7" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 8]>
-<html id="ie8" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if !(IE 6) | !(IE 7) | !(IE 8) ]><!-->
-<html dir="ltr" lang="en-US">
-<!--<![endif]-->
+<html lang="en">
<head>
- <link rel="shortcut icon" href="/favicon.ico" />
- <meta charset="UTF-8" />
- <meta name="viewport" content="width=device-width" />
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
<title>
Downloads | Apache Spark
</title>
- <link rel="stylesheet" type="text/css" media="all" href="/css/style.css" />
- <link rel="stylesheet" href="/css/pygments-default.css">
+
+
+ <!-- Bootstrap core CSS -->
+ <link href="/css/cerulean.min.css" rel="stylesheet">
+ <link href="/css/custom.css" rel="stylesheet">
<script type="text/javascript">
<!-- Google Analytics initialization -->
@@ -46,98 +39,131 @@
}
</script>
- <link rel='canonical' href='/index.html' />
-
- <style type="text/css">
- #site-title,
- #site-description {
- position: absolute !important;
- clip: rect(1px 1px 1px 1px); /* IE6, IE7 */
- clip: rect(1px, 1px, 1px, 1px);
- }
- </style>
- <style type="text/css" id="custom-background-css">
- body.custom-background { background-color: #f1f1f1; }
- </style>
+ <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
+ <![endif]-->
</head>
-<!--body class="page singular"-->
-<body class="singular">
-<div id="page" class="hfeed">
-
- <header id="branding" role="banner">
- <hgroup>
- <h1 id="site-title"><span><a href="/" title="Spark" rel="home">Spark</a></span></h1>
- <h2 id="site-description">Lightning-Fast Cluster Computing</h2>
- </hgroup>
-
- <a id="main-logo" href="/">
- <img style="height:175px; width:auto;" src="/images/spark-project-header1-cropped.png" alt="Spark: Lightning-Fast Cluster Computing" title="Spark: Lightning-Fast Cluster Computing" />
- </a>
- <div class="widget-summit">
- <a href="http://spark-summit.org"><img src="/images/Summit-Logo-FINALtr-150x150px.png" /></a>
- <div class="text">
- <a href="http://spark-summit.org/2013">
-
- <strong>Videos and Slides<br/>
- Available Now!</strong>
- </a>
- </div>
+<body>
+
+<div class="container" style="max-width: 1200px;">
+
+<div class="masthead">
+
+ <p class="lead">
+ <a href="/">
+ <img src="/images/spark-logo.png"
+ style="height:100px; width:auto; vertical-align: bottom; margin-top: 20px;"></a><span class="tagline">
+ Lightning-fast cluster computing
+ </span>
+ </p>
+
+</div>
+
+<nav class="navbar navbar-default" role="navigation">
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse"
+ data-target="#navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
</div>
- <nav id="access" role="navigation">
- <h3 class="assistive-text">Main menu</h3>
- <div class="menu-main-menu-container">
- <ul id="menu-main-menu" class="menu">
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/index.html">Home</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page current-menu-item">
- <a href="/downloads.html">Downloads</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/documentation.html">Documentation</a>
- </li>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/downloads.html">Download</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Related Projects <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Documentation <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/documentation.html">Overview</a></li>
+ <li><a href="/docs/latest/">Latest Release</a></li>
+ <li><a href="/examples.html">Examples</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Community <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/community.html">Mailing Lists</a></li>
+ <li><a href="/community.html#events">Events and Meetups</a></li>
+ <li><a href="/community.html#history">Project History</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SPARK/Powered+By+Spark">Powered By</a></li>
+ </ul>
+ </li>
+ <li><a href="/faq.html">FAQ</a></li>
+ </ul>
+ </div>
+ <!-- /.navbar-collapse -->
+</nav>
+
+
+<div class="row">
+ <div class="col-md-3 col-md-push-9">
+ <div class="news" style="margin-bottom: 20px;">
+ <h5>Latest News</h5>
+ <ul class="list-unstyled">
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/examples.html">Examples</a>
- </li>
+ <li><a href="/news/spark-0-8-1-released.html">Spark 0.8.1 released</a>
+ <span class="small">(Dec 19, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/mailing-lists.html">Mailing Lists</a>
- </li>
+ <li><a href="/news/spark-summit-2013-is-a-wrap.html">Spark Summit 2013 is a Wrap</a>
+ <span class="small">(Dec 15, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/research.html">Research</a>
- </li>
+ <li><a href="/news/announcing-the-first-spark-summit.html">Announcing the first Spark Summit: December 2, 2013</a>
+ <span class="small">(Oct 08, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/faq.html">FAQ</a>
- </li>
+ <li><a href="/news/spark-0-8-0-released.html">Spark 0.8.0 released</a>
+ <span class="small">(Sep 25, 2013)</span></li>
- </ul></div>
- </nav><!-- #access -->
-</header><!-- #branding -->
-
-
+ </ul>
+ <p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
+ </div>
+ <div class="hidden-xs hidden-sm">
+ <a href="/downloads.html" class="btn btn-success btn-lg btn-block" style="margin-bottom: 30px;">
+ Download Spark
+ </a>
+ <p style="font-size: 16px; font-weight: 500; color: #555;">
+ Related Projects:
+ </p>
+ <ul class="list-narrow">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </div>
+ </div>
- <div id="main">
- <div id="primary">
- <div id="content" role="main">
-
- <article class="page type-page status-publish hentry">
- <h3 id="download-spark">Download Spark</h3>
-<p>The latest release is Spark 0.8.1 (December 19, 2013) <a href="/releases/spark-release-0-8-1.html">(release notes)</a> (tag: <a href="https://git-wip-us.apache.org/repos/asf?p=incubator-spark.git;a=tag;h=c5167141dcc3facd053778caa9dc795bc180026a">v0.8.1-incubating</a>)</p>
+ <div class="col-md-9 col-md-pull-3">
+ <h2 id="download-spark">Download Spark</h2>
+<p>The latest release is Spark 0.8.1, released December 19, 2013 <a href="/releases/spark-release-0-8-1.html">(release notes)</a> <a href="https://git-wip-us.apache.org/repos/asf?p=incubator-spark.git;a=tag;h=c5167141dcc3facd053778caa9dc795bc180026a">(git tag)</a></p>
<ul>
<li>Download sources:
<a href="http://www.apache.org/dyn/closer.cgi/incubator/spark/spark-0.8.1-incubating/spark-0.8.1-incubating.tgz" onclick="trackOutboundLink(this, 'Release Downlaod Links', 'apache_spark-0.8.1-incubating.tgz'); return false;"> find an Apache mirror </a>
or
<a href="http://d3kbcqa49mib13.cloudfront.net/spark-0.8.1-incubating.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'cloudfront_spark-0.8.1-incubating.tgz'); return false;"> direct file download </a></li>
- <li>Download binaries for Hadoop1 (HDP1, CDH3):
+ <li>Download binaries for Hadoop 1 (HDP1, CDH3):
<a href="http://www.apache.org/dyn/closer.cgi/incubator/spark/spark-0.8.1-incubating/spark-0.8.1-incubating-bin-hadoop1.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'apache_spark-0.8.1-incubating-bin-hadoop1.tgz'); return false;"> find an Apache mirror </a>
or
<a href="http://d3kbcqa49mib13.cloudfront.net/spark-0.8.1-incubating-bin-hadoop1.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'cloudfront_spark-0.8.1-incubating-bin-hadoop1.tgz'); return false;"> direct file download </a></li>
@@ -145,15 +171,26 @@ or
<a href="http://www.apache.org/dyn/closer.cgi/incubator/spark/spark-0.8.1-incubating/spark-0.8.1-incubating-bin-cdh4.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'apache_spark-0.8.1-incubating-bin-cdh4.tgz'); return false;"> find an Apache mirror </a>
or
<a href="http://d3kbcqa49mib13.cloudfront.net/spark-0.8.1-incubating-bin-cdh4.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'cloudfront_spark-0.8.1-incubating-bin-cdh4.tgz'); return false;"> direct file download </a></li>
- <li>Download binaries for Hadoop2 (HDP2, CDH5):
+ <li>Download binaries for Hadoop 2 (HDP2, CDH5):
<a href="http://www.apache.org/dyn/closer.cgi/incubator/spark/spark-0.8.1-incubating/spark-0.8.1-incubating-bin-hadoop2.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'apache_spark-0.8.1-incubating-bin-hadoop2.tgz'); return false;"> find an Apache mirror </a>
or
<a href="http://d3kbcqa49mib13.cloudfront.net/spark-0.8.1-incubating-bin-hadoop2.tgz" onclick="trackOutboundLink(this, 'Release Download Links', 'cloudfront_spark-0.8.1-incubating-bin-hadoop2.tgz'); return false;"> direct file download </a></li>
- <li>Verify your download: <a href="http://www.apache.org/dist/incubator/spark/spark-0.8.1-incubating/">Apache signatures and checksums</a></li>
+ <li>Verify your download: <a href="http://www.apache.org/dist/incubator/spark/spark-0.8.1-incubating/">signatures and checksums</a></li>
</ul>
-<h3 id="link-against-spark">Link Against Spark</h3>
-<p>Spark artifacts are <a href="http://search.maven.org/#browse%7C1686516968">hosted in Maven Central</a>. You can depend on them at the following coordinates:</p>
+<!--
+The Spark download includes
+<a href="/streaming/">Spark Streaming</a>,
+<a href="/mllib/">MLlib</a>, and
+<a href="http://amplab.github.io/graphx/">GraphX</a>.
+-->
+
+<p>The Spark download includes
+<a href="/streaming/">Spark Streaming</a> and
+<a href="/mllib/">MLlib</a>.</p>
+
+<h3 id="link-with-spark">Link with Spark</h3>
+<p>Spark artifacts are <a href="http://search.maven.org/#browse%7C1686516968">hosted in Maven Central</a>. You can add a Maven dependency with the following coordinates:</p>
<pre><code>groupId: org.apache.spark
artifactId: spark-core_2.9.3
@@ -180,31 +217,31 @@ version: 0.8.1-incubating
<li><a href="http://spark-project.org/download-spark-0.6.0-sources-tgz">Spark 0.6.0</a> (October 15, 2012) <a href="/releases/spark-release-0-6-0.html">(release notes)</a></li>
<li><a href="http://spark-project.org/download-spark-0.5.1-sources-tgz">Spark 0.5.1</a> (October 11, 2012) <a href="/releases/spark-release-0-5-1.html">(release notes)</a></li>
<li><a href="http://spark-project.org/download-spark-0.5.0-sources-tgz">Spark 0.5.0</a> (June 12, 2012) <a href="/releases/spark-release-0-5-0.html">(release notes)</a></li>
- <li><a href=""http://spark-project.org/download-spark-0.3-for-scala-2-9-sources-tgz"">Spark 0.3 for Scala 2.9</a> (July 14, 2011) <a href="/releases/spark-release-0-3.html">release notes</a></li>
+ <li><a href=""http://spark-project.org/download-spark-0.3-for-scala-2-9-sources-tgz"">Spark 0.3 for Scala 2.9</a> (July 14, 2011) <a href="/releases/spark-release-0-3.html">(release notes)</a></li>
<li><a href="http://spark-project.org/download-spark-0.3-for-scala-2-8-sources-tgz">Spark 0.3 for Scala 2.8</a> (July 14, 2011) <a href="/releases/spark-release-0-3.html">(release notes)</a></li>
<li><a href="http://spark-project.org/download-spark-0.2-sources-tgz">Spark 0.2</a> (May 24, 2011)</li>
</ul>
- </article><!-- #post -->
-
- </div><!-- #content -->
-
- <footer id="colophon" role="contentinfo">
- <div id="site-generator">
- <p style="padding-top: 0; padding-bottom: 15px;">
- Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
- <a href="http://incubator.apache.org/" style="border: none;">
- <img style="vertical-align: middle; border: none;" src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
- </a>
- </p>
</div>
-</footer><!-- #colophon -->
+</div>
+
+
+
+<footer class="small">
+ <hr>
+ Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
+ <a href="http://incubator.apache.org/" style="border: none;">
+ <img style="vertical-align: middle; float: right; margin-bottom: 15px;"
+ src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
+ </a>
+</footer>
- </div><!-- #primary -->
- </div><!-- #main -->
-</div><!-- #page -->
+</div>
+<script src="https://code.jquery.com/jquery.js"></script>
+<script src="//netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js"></script>
+<script src="/js/lang-tabs.js"></script>
</body>
</html>
Modified: incubator/spark/site/examples.html
URL: http://svn.apache.org/viewvc/incubator/spark/site/examples.html?rev=1560502&r1=1560501&r2=1560502&view=diff
==============================================================================
--- incubator/spark/site/examples.html (original)
+++ incubator/spark/site/examples.html Wed Jan 22 20:33:24 2014
@@ -1,27 +1,20 @@
<!DOCTYPE html>
-<!--[if IE 6]>
-<html id="ie6" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 7]>
-<html id="ie7" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 8]>
-<html id="ie8" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if !(IE 6) | !(IE 7) | !(IE 8) ]><!-->
-<html dir="ltr" lang="en-US">
-<!--<![endif]-->
+<html lang="en">
<head>
- <link rel="shortcut icon" href="/favicon.ico" />
- <meta charset="UTF-8" />
- <meta name="viewport" content="width=device-width" />
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
<title>
Examples | Apache Spark
</title>
- <link rel="stylesheet" type="text/css" media="all" href="/css/style.css" />
- <link rel="stylesheet" href="/css/pygments-default.css">
+
+
+ <!-- Bootstrap core CSS -->
+ <link href="/css/cerulean.min.css" rel="stylesheet">
+ <link href="/css/custom.css" rel="stylesheet">
<script type="text/javascript">
<!-- Google Analytics initialization -->
@@ -46,128 +39,212 @@
}
</script>
- <link rel='canonical' href='/index.html' />
-
- <style type="text/css">
- #site-title,
- #site-description {
- position: absolute !important;
- clip: rect(1px 1px 1px 1px); /* IE6, IE7 */
- clip: rect(1px, 1px, 1px, 1px);
- }
- </style>
- <style type="text/css" id="custom-background-css">
- body.custom-background { background-color: #f1f1f1; }
- </style>
+ <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
+ <![endif]-->
</head>
-<!--body class="page singular"-->
-<body class="page singular">
-<div id="page" class="hfeed">
-
- <header id="branding" role="banner">
- <hgroup>
- <h1 id="site-title"><span><a href="/" title="Spark" rel="home">Spark</a></span></h1>
- <h2 id="site-description">Lightning-Fast Cluster Computing</h2>
- </hgroup>
-
- <a id="main-logo" href="/">
- <img style="height:175px; width:auto;" src="/images/spark-project-header1-cropped.png" alt="Spark: Lightning-Fast Cluster Computing" title="Spark: Lightning-Fast Cluster Computing" />
- </a>
- <div class="widget-summit">
- <a href="http://spark-summit.org"><img src="/images/Summit-Logo-FINALtr-150x150px.png" /></a>
- <div class="text">
- <a href="http://spark-summit.org/2013">
-
- <strong>Videos and Slides<br/>
- Available Now!</strong>
- </a>
- </div>
+<body>
+
+<div class="container" style="max-width: 1200px;">
+
+<div class="masthead">
+
+ <p class="lead">
+ <a href="/">
+ <img src="/images/spark-logo.png"
+ style="height:100px; width:auto; vertical-align: bottom; margin-top: 20px;"></a><span class="tagline">
+ Lightning-fast cluster computing
+ </span>
+ </p>
+
+</div>
+
+<nav class="navbar navbar-default" role="navigation">
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse"
+ data-target="#navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
</div>
- <nav id="access" role="navigation">
- <h3 class="assistive-text">Main menu</h3>
- <div class="menu-main-menu-container">
- <ul id="menu-main-menu" class="menu">
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/index.html">Home</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/downloads.html">Downloads</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/documentation.html">Documentation</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page current-menu-item">
- <a href="/examples.html">Examples</a>
- </li>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/downloads.html">Download</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Related Projects <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Documentation <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/documentation.html">Overview</a></li>
+ <li><a href="/docs/latest/">Latest Release</a></li>
+ <li><a href="/examples.html">Examples</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Community <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/community.html">Mailing Lists</a></li>
+ <li><a href="/community.html#events">Events and Meetups</a></li>
+ <li><a href="/community.html#history">Project History</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SPARK/Powered+By+Spark">Powered By</a></li>
+ </ul>
+ </li>
+ <li><a href="/faq.html">FAQ</a></li>
+ </ul>
+ </div>
+ <!-- /.navbar-collapse -->
+</nav>
+
+
+<div class="row">
+ <div class="col-md-3 col-md-push-9">
+ <div class="news" style="margin-bottom: 20px;">
+ <h5>Latest News</h5>
+ <ul class="list-unstyled">
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/mailing-lists.html">Mailing Lists</a>
- </li>
+ <li><a href="/news/spark-0-8-1-released.html">Spark 0.8.1 released</a>
+ <span class="small">(Dec 19, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/research.html">Research</a>
- </li>
+ <li><a href="/news/spark-summit-2013-is-a-wrap.html">Spark Summit 2013 is a Wrap</a>
+ <span class="small">(Dec 15, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/faq.html">FAQ</a>
- </li>
+ <li><a href="/news/announcing-the-first-spark-summit.html">Announcing the first Spark Summit: December 2, 2013</a>
+ <span class="small">(Oct 08, 2013)</span></li>
- </ul></div>
- </nav><!-- #access -->
-</header><!-- #branding -->
-
-
-
- <div id="main">
- <div id="primary">
- <div id="content" role="main">
+ <li><a href="/news/spark-0-8-0-released.html">Spark 0.8.0 released</a>
+ <span class="small">(Sep 25, 2013)</span></li>
- <article class="page type-page status-publish hentry">
- <h2>Spark Examples</h2>
+ </ul>
+ <p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
+ </div>
+ <div class="hidden-xs hidden-sm">
+ <a href="/downloads.html" class="btn btn-success btn-lg btn-block" style="margin-bottom: 30px;">
+ Download Spark
+ </a>
+ <p style="font-size: 16px; font-weight: 500; color: #555;">
+ Related Projects:
+ </p>
+ <ul class="list-narrow">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </div>
+ </div>
-<p>Spark is built around <em>distributed datasets</em> that support types of parallel operations: transformations, which are lazy and yield another distributed dataset (e.g., <code>map</code>, <code>filter</code>, and <code>join</code>), and actions, which force the computation of a dataset and return a result (e.g., <code>count</code>). The following examples show off some of the available operations and features. Several additional examples are distributed with Spark:</p>
+ <div class="col-md-9 col-md-pull-3">
+ <h2>Spark Examples</h2>
-<ul>
- <li>Core Spark: <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/scala/org/apache/spark/examples">Scala examples</a>, <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/java/org/apache/spark/examples">Java examples</a>, <a href="https://github.com/apache/incubator-spark/tree/master/python/examples">Python examples</a></li>
- <li>Streaming Spark: <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/scala/org/apache/spark/streaming/examples">Scala examples</a>, <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/java/org/apache/spark/streaming/examples">Java examples</a></li>
-</ul>
+<p>These examples give a quick overview of the Spark API.
+Spark is built on the concept of <em>distributed datasets</em>, which contain arbitrary Java or
+Python objects. You create a dataset from external data, then apply parallel operations
+to it. There are two types of operations: <em>transformations</em>, which define a new dataset based on
+previous ones, and <em>actions</em>, which kick off a job to execute on a cluster.</p>
<h3>Text Search</h3>
<p>In this example, we search through the error messages in a log file:</p>
-<p>
-</p>
-<div class="code">
-<span class="keyword">val</span> file = spark.textFile(<span class="string">"hdfs://..."</span>)<br />
-<span class="keyword">val</span> errors = file.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("ERROR")</span>)<br />
-<span class="comment">// Count all the errors</span><br />
-errors.<span class="sparkop">count</span>()<br />
-<span class="comment">// Count errors mentioning MySQL</span><br />
-errors.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("MySQL")</span>).<span class="sparkop">count</span>()<br />
-<span class="comment">// Fetch the MySQL errors as an array of strings</span><br />
-errors.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("MySQL")</span>).<span class="sparkop">collect</span>()<br />
+<ul class="nav nav-tabs">
+ <li class="lang-tab lang-tab-scala active"><a href="#">Scala</a></li>
+ <li class="lang-tab lang-tab-java"><a href="#">Java</a></li>
+ <li class="lang-tab lang-tab-python"><a href="#">Python</a></li>
+</ul>
+<div class="tab-content">
+ <div class="tab-pane tab-pane-scala active">
+ <div class="code code-tab">
+ <span class="keyword">val</span> file = spark.textFile(<span class="string">"hdfs://..."</span>)<br />
+ <span class="keyword">val</span> errors = file.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("ERROR")</span>)<br />
+ <span class="comment">// Count all the errors</span><br />
+ errors.<span class="sparkop">count</span>()<br />
+ <span class="comment">// Count errors mentioning MySQL</span><br />
+ errors.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("MySQL")</span>).<span class="sparkop">count</span>()<br />
+ <span class="comment">// Fetch the MySQL errors as an array of strings</span><br />
+ errors.<span class="sparkop">filter</span>(<span class="closure">line => line.contains("MySQL")</span>).<span class="sparkop">collect</span>()<br />
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-java">
+ <div class="code code-tab">
+ JavaRDD<String> file = spark.textFile(<span class="string">"hdfs://..."</span>);<br />
+ JavaRDD<String> errors = file.<span class="sparkop">filter</span>(<span class="closure">new Function<String, Boolean>() {<br />
+ public Boolean call(String s) { return s.contains("ERROR"); }<br />
+ }</span>);<br />
+ <span class="comment">// Count all the errors</span><br />
+ errors.<span class="sparkop">count</span>();<br />
+ <span class="comment">// Count errors mentioning MySQL</span><br />
+ errors.<span class="sparkop">filter</span>(<span class="closure">new Function<String, Boolean>() {<br />
+ public Boolean call(String s) { return s.contains("MySQL"); }<br />
+ }</span>).<span class="sparkop">count</span>();<br />
+ <span class="comment">// Fetch the MySQL errors as an array of strings</span><br />
+ errors.<span class="sparkop">filter</span>(<span class="closure">new Function<String, Boolean>() {<br />
+ public Boolean call(String s) { return s.contains("MySQL"); }<br />
+ }</span>).<span class="sparkop">collect</span>();<br />
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-python">
+ <div class="code code-tab">
+ file = spark.textFile(<span class="string">"hdfs://..."</span>)<br />
+ errors = file.<span class="sparkop">filter</span>(<span class="closure">lambda line: "ERROR" in line</span>)<br />
+ <span class="comment"># Count all the errors</span><br />
+ errors.<span class="sparkop">count</span>()<br />
+ <span class="comment"># Count errors mentioning MySQL</span><br />
+ errors.<span class="sparkop">filter</span>(<span class="closure">lambda line: "MySQL" in line</span>).<span class="sparkop">count</span>()<br />
+ <span class="comment"># Fetch the MySQL errors as an array of strings</span><br />
+ errors.<span class="sparkop">filter</span>(<span class="closure">lambda line: "MySQL" in line</span>).<span class="sparkop">collect</span>()<br />
+ </div>
+ </div>
</div>
-<p></p>
-<p>The red code fragments are Scala function literals (closures) that get passed automatically to the cluster. The blue ones are Spark operations.</p>
+<p>The red code fragments are function literals (closures) that get passed automatically to the cluster. The blue ones are Spark operations.</p>
<h3>In-Memory Text Search</h3>
<p>Spark can <em>cache</em> datasets in memory to speed up reuse. In the example above, we can load just the error messages in RAM using:</p>
-<p>
-</p>
-<div class="code">
-errors.<span class="sparkop">cache</span>()
+<ul class="nav nav-tabs">
+ <li class="lang-tab lang-tab-scala active"><a href="#">Scala</a></li>
+ <li class="lang-tab lang-tab-java"><a href="#">Java</a></li>
+ <li class="lang-tab lang-tab-python"><a href="#">Python</a></li>
+</ul>
+<div class="tab-content">
+ <div class="tab-pane tab-pane-scala active">
+ <div class="code code-tab">
+ errors.<span class="sparkop">cache</span>()
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-java">
+ <div class="code code-tab">
+ errors.<span class="sparkop">cache</span>();
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-python">
+ <div class="code code-tab">
+ errors.<span class="sparkop">cache</span>()
+ </div>
+ </div>
</div>
-<p></p>
<p>After the first action that uses <code>errors</code>, later ones will be much faster.</p>
@@ -175,80 +252,185 @@ errors.<span class="sparkop">cache</span
<p>In this example, we use a few more transformations to build a dataset of (String, Int) pairs called <code>counts</code> and then save it to a file.</p>
-<p>
-</p>
-<div class="code">
-<span class="keyword">val</span> file = spark.textFile(<span class="string">"hdfs://..."</span>)<br />
-<span class="keyword">val</span> counts = file.<span class="sparkop">flatMap</span>(<span class="closure">line => line.split(" ")</span>)<br />
- .<span class="sparkop">map</span>(<span class="closure">word => (word, 1)</span>)<br />
- .<span class="sparkop">reduceByKey</span>(<span class="closure">_ + _</span>)<br />
-counts.<span class="sparkop">saveAsTextFile</span>(<span class="string">"hdfs://..."</span>)
+<ul class="nav nav-tabs">
+ <li class="lang-tab lang-tab-scala active"><a href="#">Scala</a></li>
+ <li class="lang-tab lang-tab-java"><a href="#">Java</a></li>
+ <li class="lang-tab lang-tab-python"><a href="#">Python</a></li>
+</ul>
+<div class="tab-content">
+ <div class="tab-pane tab-pane-scala active">
+ <div class="code code-tab">
+ <span class="keyword">val</span> file = spark.textFile(<span class="string">"hdfs://..."</span>)<br />
+ <span class="keyword">val</span> counts = file.<span class="sparkop">flatMap</span>(<span class="closure">line => line.split(" ")</span>)<br />
+ .<span class="sparkop">map</span>(<span class="closure">word => (word, 1)</span>)<br />
+ .<span class="sparkop">reduceByKey</span>(<span class="closure">_ + _</span>)<br />
+ counts.<span class="sparkop">saveAsTextFile</span>(<span class="string">"hdfs://..."</span>)
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-java">
+ <div class="code code-tab">
+ JavaRDD<String> file = spark.textFile(<span class="string">"hdfs://..."</span>);<br />
+ JavaRDD<String> words = file.<span class="sparkop">flatMap</span>(<span class="closure">new FlatMapFunction<String, String>()<br />
+ public Iterable<String> call(String s) { return Arrays.asList(s.split(" ")); }<br />
+ }</span>);<br />
+ JavaPairRDD<String, Integer> pairs = words.<span class="sparkop">map</span>(<span class="closure">new PairFunction<String, String, Integer>()<br />
+ public Tuple2<String, Integer> call(String s) { return new Tuple2<String, Integer>(s, 1); }<br />
+ }</span>);<br />
+ JavaPairRDD<String, Integer> counts = pairs.<span class="sparkop">reduceByKey</span>(<span class="closure">new Function2<Integer, Integer>()<br />
+ public Integer call(Integer a, Integer b) { return a + b; }<br />
+ }</span>);<br />
+ counts.<span class="sparkop">saveAsTextFile</span>(<span class="string">"hdfs://..."</span>);
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-python">
+ <div class="code code-tab">
+ file = spark.textFile(<span class="string">"hdfs://..."</span>)<br />
+ counts = file.<span class="sparkop">flatMap</span>(<span class="closure">lambda line: line.split(" ")</span>) \<br />
+ .<span class="sparkop">map</span>(<span class="closure">lambda word: (word, 1)</span>) \<br />
+ .<span class="sparkop">reduceByKey</span>(<span class="closure">lambda a, b: a + b</span>)<br />
+ counts.<span class="sparkop">saveAsTextFile</span>(<span class="string">"hdfs://..."</span>)
+ </div>
+ </div>
</div>
-<p></p>
<h3>Estimating Pi</h3>
<p>Spark can also be used for compute-intensive tasks. This code estimates <span style="font-family: serif; font-size: 120%;">Ï</span> by "throwing darts" at a circle. We pick random points in the unit square ((0, 0) to (1,1)) and see how many fall in the unit circle. The fraction should be <span style="font-family: serif; font-size: 120%;">Ï / 4</span>, so we use this to get our estimate.</p>
-<p>
-</p>
-<div class="code">
-<span class="keyword">val</span> count = spark.parallelize(1 to NUM_SAMPLES).<span class="sparkop">map</span>(<span class="closure">i =><br />
- <span class="keyword">val</span> x = Math.random<br />
- <span class="keyword">val</span> y = Math.random<br />
- <span class="keyword">if</span> (x*x + y*y < 1) 1.0 <span class="keyword">else</span> 0.0<br />
-</span>).<span class="sparkop">reduce</span>(<span class="closure">_ + _</span>)<br />
-println(<span class="string">"Pi is roughly "</span> + 4 * count / NUM_SAMPLES)<br />
+<ul class="nav nav-tabs">
+ <li class="lang-tab lang-tab-scala active"><a href="#">Scala</a></li>
+ <li class="lang-tab lang-tab-java"><a href="#">Java</a></li>
+ <li class="lang-tab lang-tab-python"><a href="#">Python</a></li>
+</ul>
+<div class="tab-content">
+ <div class="tab-pane tab-pane-scala active">
+ <div class="code code-tab">
+ <span class="keyword">val</span> count = spark.parallelize(1 to NUM_SAMPLES).<span class="sparkop">map</span>(<span class="closure">i =><br />
+ val x = Math.random()<br />
+ val y = Math.random()<br />
+ if (x*x + y*y < 1) 1 else 0<br />
+ </span>).<span class="sparkop">reduce</span>(<span class="closure">_ + _</span>)<br />
+ println(<span class="string">"Pi is roughly "</span> + 4.0 * count / NUM_SAMPLES)<br />
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-java">
+ <div class="code code-tab">
+ <span class="keyword">int</span> count = spark.parallelize(makeRange(1, NUM_SAMPLES)).<span class="sparkop">filter</span>(<span class="closure">new Function<Integer, Boolean>() {<br />
+ public Integer call(Integer i) {<br />
+ double x = Math.random();<br />
+ double y = Math.random();<br />
+ return x*x + y*y < 1;<br />
+ }<br />
+ }</span>).<span class="sparkop">count</span>();<br />
+ System.out.println(<span class="string">"Pi is roughly "</span> + 4 * count / NUM_SAMPLES);<br />
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-python">
+ <div class="code code-tab">
+ <span class="keyword">def</span> sample(p):<br />
+ x, y = random(), random()<br />
+ <span class="keyword">return</span> 1 <span class="keyword">if</span> x*x + y*y < 1 <span class="keyword">else</span> 0<br /><br />
+ count = spark.parallelize(xrange(0, NUM_SAMPLES)).<span class="sparkop">map</span>(<span class="closure">sample</span>) \<br />
+ .<span class="sparkop">reduce</span>(<span class="closure">lambda a, b: a + b</span>)<br />
+ print <span class="string">"Pi is roughly %f"</span> % (4.0 * count / NUM_SAMPLES)<br />
+ </div>
+ </div>
</div>
-<p></p>
<h3>Logistic Regression</h3>
-<p>This is an iterative machine learning algorithm that seeks to find the best hyperplane that separates two sets of points in a multi-dimensional feature space. It can be used to classify messages into spam vs non-spam, for example. Because the algorithm applies the same MapReduce operation repeatedly to the same dataset, it benefits greatly from caching the input data in RAM across iterations.</p>
+<p>This is an iterative machine learning algorithm that seeks to find the best hyperplane that separates two sets of points in a multi-dimensional feature space. It can be used to classify messages into spam vs non-spam, for example. Because the algorithm applies the same MapReduce operation repeatedly to the same dataset, it benefits greatly from caching the input in RAM across iterations.</p>
-<p>
-</p>
-<div class="code">
-<span class="keyword">val</span> points = spark.textFile(...).<span class="sparkop">map</span>(parsePoint).<span class="sparkop">cache</span>()<br />
-<span class="keyword">var</span> w = Vector.random(D) <span class="comment">// current separating plane</span><br />
-<span class="keyword">for</span> (i <- 1 to ITERATIONS) {<br />
- <span class="keyword">val</span> gradient = points.<span class="sparkop">map</span>(<span class="closure">p =><br />
- (1 / (1 + exp(-p.y*(w dot p.x))) - 1) * p.y * p.x<br />
- </span>).<span class="sparkop">reduce</span>(<span class="closure">_ + _</span>)<br />
- w -= gradient<br />
-}<br />
-println(<span class="string">"Final separating plane: "</span> + w)<br />
+<ul class="nav nav-tabs">
+ <li class="lang-tab lang-tab-scala active"><a href="#">Scala</a></li>
+ <li class="lang-tab lang-tab-java"><a href="#">Java</a></li>
+ <li class="lang-tab lang-tab-python"><a href="#">Python</a></li>
+</ul>
+<div class="tab-content">
+ <div class="tab-pane tab-pane-scala active">
+ <div class="code code-tab">
+ <span class="keyword">val</span> points = spark.textFile(...).<span class="sparkop">map</span>(parsePoint).<span class="sparkop">cache</span>()<br />
+ <span class="keyword">var</span> w = Vector.random(D) <span class="comment">// current separating plane</span><br />
+ <span class="keyword">for</span> (i <- 1 to ITERATIONS) {<br />
+ <span class="keyword">val</span> gradient = points.<span class="sparkop">map</span>(<span class="closure">p =><br />
+ (1 / (1 + exp(-p.y*(w dot p.x))) - 1) * p.y * p.x<br />
+ </span>).<span class="sparkop">reduce</span>(<span class="closure">_ + _</span>)<br />
+ w -= gradient<br />
+ }<br />
+ println(<span class="string">"Final separating plane: "</span> + w)<br />
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-java">
+ <div class="code code-tab">
+ <span class="keyword">class</span> ComputeGradient <span class="keyword">extends</span> Function<DataPoint, Vector> {<br />
+ <span class="keyword">private</span> Vector w;<br />
+ ComputeGradient(Vector w) { <span class="keyword">this</span>.w = w; }<br />
+ <span class="keyword">public</span> Vector call(DataPoint p) {<br />
+ <span class="keyword">return</span> p.x.times(p.y * (1 / (1 + Math.exp(w.dot(p.x))) - 1));<br />
+ }<br />
+ }<br />
+ <br />
+ JavaRDD<DataPoint> points = spark.textFile(...).<span class="sparkop">map</span>(<span class="closure">new ParsePoint()</span>).<span class="sparkop">cache</span>();<br />
+ Vector w = Vector.random(D); <span class="comment">// current separating plane</span><br />
+ <span class="keyword">for</span> (<span class="keyword">int</span> i = 0; i < ITERATIONS; i++) {<br />
+ Vector gradient = points.<span class="sparkop">map</span>(<span class="closure">new ComputeGradient(w)</span>).<span class="sparkop">reduce</span>(<span class="closure">new AddVectors()</span>);<br />
+ w = w.subtract(gradient);<br />
+ }<br />
+ System.out.println(<span class="string">"Final separating plane: "</span> + w);<br />
+ </div>
+ </div>
+ <div class="tab-pane tab-pane-python">
+ <div class="code code-tab">
+ points = spark.textFile(...).<span class="sparkop">map</span>(parsePoint).<span class="sparkop">cache</span>()<br />
+ w = numpy.random.ranf(size = D) <span class="comment"># current separating plane</span><br />
+ <span class="keyword">for</span> i <span class="keyword">in</span> range(ITERATIONS):<br />
+ gradient = points.<span class="sparkop">map</span>(<span class="closure"><br />
+ lambda p: (1 / (1 + exp(-p.y*(w.dot(p.x)))) - 1) * p.y * p.x<br />
+ </span>).<span class="sparkop">reduce</span>(<span class="closure">lambda a, b: a + b</span>)<br />
+ w -= gradient<br />
+ print <span class="string">"Final separating plane: %s"</span> % w<br />
+ </div>
+ </div>
</div>
-<p></p>
-<p>Note that <code>w</code> gets shipped automatically to the cluster with every <code>map</code> call.</p>
+<p>Note that the current separating plane, <code>w</code>, gets shipped automatically to the cluster with every <code>map</code> call.</p>
-<p>The graph below compares the performance of this Spark program against a Hadoop implementation on 30 GB of data on an 80-core cluster, showing the benefit of in-memory caching:</p>
+<p>The graph below compares the running time per iteration of this Spark program against a Hadoop implementation on 100 GB of data on a 100-node cluster, showing the benefit of in-memory caching:</p>
-<p align="center">
-<img src="/images/spark-lr.png" alt="Logistic regression performance in Spark vs Hadoop" />
+<p style="margin-top: 20px; margin-bottom: 30px;">
+<img src="/images/logistic-regression.png" alt="Logistic regression performance in Spark vs Hadoop" />
</p>
+<p><a name="additional"></a></p>
+<h2>Additional Examples</h2>
+
+<p>Many additional examples are distributed with Spark:</p>
+
+<ul>
+ <li>Basic Spark: <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/scala/org/apache/spark/examples">Scala examples</a>, <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/java/org/apache/spark/examples">Java examples</a>, <a href="https://github.com/apache/incubator-spark/tree/master/python/examples">Python examples</a></li>
+ <li>Spark Streaming: <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/scala/org/apache/spark/streaming/examples">Scala examples</a>, <a href="https://github.com/apache/incubator-spark/tree/master/examples/src/main/java/org/apache/spark/streaming/examples">Java examples</a></li>
+</ul>
+
- </article><!-- #post -->
-
- </div><!-- #content -->
-
- <footer id="colophon" role="contentinfo">
- <div id="site-generator">
- <p style="padding-top: 0; padding-bottom: 15px;">
- Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
- <a href="http://incubator.apache.org/" style="border: none;">
- <img style="vertical-align: middle; border: none;" src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
- </a>
- </p>
</div>
-</footer><!-- #colophon -->
+</div>
- </div><!-- #primary -->
- </div><!-- #main -->
-</div><!-- #page -->
+<footer class="small">
+ <hr>
+ Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
+ <a href="http://incubator.apache.org/" style="border: none;">
+ <img style="vertical-align: middle; float: right; margin-bottom: 15px;"
+ src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
+ </a>
+</footer>
+
+</div>
+
+<script src="https://code.jquery.com/jquery.js"></script>
+<script src="//netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js"></script>
+<script src="/js/lang-tabs.js"></script>
+
</body>
</html>
Modified: incubator/spark/site/faq.html
URL: http://svn.apache.org/viewvc/incubator/spark/site/faq.html?rev=1560502&r1=1560501&r2=1560502&view=diff
==============================================================================
--- incubator/spark/site/faq.html (original)
+++ incubator/spark/site/faq.html Wed Jan 22 20:33:24 2014
@@ -1,27 +1,20 @@
<!DOCTYPE html>
-<!--[if IE 6]>
-<html id="ie6" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 7]>
-<html id="ie7" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if IE 8]>
-<html id="ie8" dir="ltr" lang="en-US">
-<![endif]-->
-<!--[if !(IE 6) | !(IE 7) | !(IE 8) ]><!-->
-<html dir="ltr" lang="en-US">
-<!--<![endif]-->
+<html lang="en">
<head>
- <link rel="shortcut icon" href="/favicon.ico" />
- <meta charset="UTF-8" />
- <meta name="viewport" content="width=device-width" />
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
<title>
FAQ | Apache Spark
</title>
- <link rel="stylesheet" type="text/css" media="all" href="/css/style.css" />
- <link rel="stylesheet" href="/css/pygments-default.css">
+
+
+ <!-- Bootstrap core CSS -->
+ <link href="/css/cerulean.min.css" rel="stylesheet">
+ <link href="/css/custom.css" rel="stylesheet">
<script type="text/javascript">
<!-- Google Analytics initialization -->
@@ -46,141 +39,190 @@
}
</script>
- <link rel='canonical' href='/index.html' />
-
- <style type="text/css">
- #site-title,
- #site-description {
- position: absolute !important;
- clip: rect(1px 1px 1px 1px); /* IE6, IE7 */
- clip: rect(1px, 1px, 1px, 1px);
- }
- </style>
- <style type="text/css" id="custom-background-css">
- body.custom-background { background-color: #f1f1f1; }
- </style>
+ <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+ <!--[if lt IE 9]>
+ <script src="https://oss.maxcdn.com/libs/html5shiv/3.7.0/html5shiv.js"></script>
+ <script src="https://oss.maxcdn.com/libs/respond.js/1.3.0/respond.min.js"></script>
+ <![endif]-->
</head>
-<!--body class="page singular"-->
-<body class="page singular">
-<div id="page" class="hfeed">
-
- <header id="branding" role="banner">
- <hgroup>
- <h1 id="site-title"><span><a href="/" title="Spark" rel="home">Spark</a></span></h1>
- <h2 id="site-description">Lightning-Fast Cluster Computing</h2>
- </hgroup>
-
- <a id="main-logo" href="/">
- <img style="height:175px; width:auto;" src="/images/spark-project-header1-cropped.png" alt="Spark: Lightning-Fast Cluster Computing" title="Spark: Lightning-Fast Cluster Computing" />
- </a>
- <div class="widget-summit">
- <a href="http://spark-summit.org"><img src="/images/Summit-Logo-FINALtr-150x150px.png" /></a>
- <div class="text">
- <a href="http://spark-summit.org/2013">
-
- <strong>Videos and Slides<br/>
- Available Now!</strong>
- </a>
- </div>
+<body>
+
+<div class="container" style="max-width: 1200px;">
+
+<div class="masthead">
+
+ <p class="lead">
+ <a href="/">
+ <img src="/images/spark-logo.png"
+ style="height:100px; width:auto; vertical-align: bottom; margin-top: 20px;"></a><span class="tagline">
+ Lightning-fast cluster computing
+ </span>
+ </p>
+
+</div>
+
+<nav class="navbar navbar-default" role="navigation">
+ <!-- Brand and toggle get grouped for better mobile display -->
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse"
+ data-target="#navbar-collapse-1">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
</div>
- <nav id="access" role="navigation">
- <h3 class="assistive-text">Main menu</h3>
- <div class="menu-main-menu-container">
- <ul id="menu-main-menu" class="menu">
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/index.html">Home</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/downloads.html">Downloads</a>
- </li>
-
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/documentation.html">Documentation</a>
- </li>
+ <!-- Collect the nav links, forms, and other content for toggling -->
+ <div class="collapse navbar-collapse" id="navbar-collapse-1">
+ <ul class="nav navbar-nav">
+ <li><a href="/downloads.html">Download</a></li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Related Projects <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Documentation <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/documentation.html">Overview</a></li>
+ <li><a href="/docs/latest/">Latest Release</a></li>
+ <li><a href="/examples.html">Examples</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">
+ Community <b class="caret"></b>
+ </a>
+ <ul class="dropdown-menu">
+ <li><a href="/community.html">Mailing Lists</a></li>
+ <li><a href="/community.html#events">Events and Meetups</a></li>
+ <li><a href="/community.html#history">Project History</a></li>
+ <li><a href="https://cwiki.apache.org/confluence/display/SPARK/Powered+By+Spark">Powered By</a></li>
+ </ul>
+ </li>
+ <li><a href="/faq.html">FAQ</a></li>
+ </ul>
+ </div>
+ <!-- /.navbar-collapse -->
+</nav>
+
+
+<div class="row">
+ <div class="col-md-3 col-md-push-9">
+ <div class="news" style="margin-bottom: 20px;">
+ <h5>Latest News</h5>
+ <ul class="list-unstyled">
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/examples.html">Examples</a>
- </li>
+ <li><a href="/news/spark-0-8-1-released.html">Spark 0.8.1 released</a>
+ <span class="small">(Dec 19, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/mailing-lists.html">Mailing Lists</a>
- </li>
+ <li><a href="/news/spark-summit-2013-is-a-wrap.html">Spark Summit 2013 is a Wrap</a>
+ <span class="small">(Dec 15, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page ">
- <a href="/research.html">Research</a>
- </li>
+ <li><a href="/news/announcing-the-first-spark-summit.html">Announcing the first Spark Summit: December 2, 2013</a>
+ <span class="small">(Oct 08, 2013)</span></li>
- <li class="menu-item menu-item-type-post_type menu-item-object-page current-menu-item">
- <a href="/faq.html">FAQ</a>
- </li>
+ <li><a href="/news/spark-0-8-0-released.html">Spark 0.8.0 released</a>
+ <span class="small">(Sep 25, 2013)</span></li>
- </ul></div>
- </nav><!-- #access -->
-</header><!-- #branding -->
-
-
+ </ul>
+ <p class="small" style="text-align: right;"><a href="/news/index.html">Archive</a></p>
+ </div>
+ <div class="hidden-xs hidden-sm">
+ <a href="/downloads.html" class="btn btn-success btn-lg btn-block" style="margin-bottom: 30px;">
+ Download Spark
+ </a>
+ <p style="font-size: 16px; font-weight: 500; color: #555;">
+ Related Projects:
+ </p>
+ <ul class="list-narrow">
+ <li><a href="http://shark.cs.berkeley.edu">Shark (SQL)</a></li>
+ <li><a href="/streaming/">Spark Streaming</a></li>
+ <li><a href="/mllib/">MLlib (machine learning)</a></li>
+ <li><a href="http://amplab.github.io/graphx/">GraphX (graph)</a></li>
+ </ul>
+ </div>
+ </div>
- <div id="main">
- <div id="primary">
- <div id="content" role="main">
-
- <article class="page type-page status-publish hentry">
- <h2>Spark FAQ</h2>
+ <div class="col-md-9 col-md-pull-3">
+ <h2>Spark FAQ</h2>
-<p class="question">Is Spark a modified version of Hadoop?</p>
-<p class="answer">No. Spark is a completely separate codebase optimized for low latency, although it can load data from any Hadoop input source (InputFormat).</p>
+<p class="question">How does Spark relate to Hadoop?</p>
+<p class="answer">
+Spark is a fast and powerful engine for processing Hadoop data.
+It runs in Hadoop clusters through
+<a href="http://hadoop.apache.org/docs/current2/hadoop-yarn/hadoop-yarn-site/YARN.html">Hadoop YARN</a>
+or Spark's <a href="/docs/latest/spark-standalone.html">standalone mode</a>, and it can process
+data in HDFS, HBase, Cassandra, Hive, and any Hadoop InputFormat.
+It is designed to perform both general data processing (similar to MapReduce) and new workloads like
+streaming, interactive queries, and machine learning.
+</p>
<p class="question">Which languages does Spark support?</p>
-<p class="answer">Starting in version 0.7, Spark supports Scala, Java and Python.</p>
+<p class="answer">Spark supports Scala, Java and Python.</p>
<p class="question">Does Spark require modified versions of Scala or Python?</p>
<p class="answer">No. Spark requires no changes to Scala or compiler plugins. The Python API uses the standard CPython implementation, and can call into existing C libraries for Python such as NumPy.</p>
<p class="question">What happens when a cached dataset does not fit in memory?</p>
-<p class="answer">Spark can either spill it to disk or recompute the partitions that don't fit in RAM each time they are requested. By default, it uses recomputation, but you can set a dataset's <a href="/docs/latest/scala-programming-guide.html#rdd-persistence">storage level</a> to <tt>MEMORY_AND_DISK</tt> to avoid this. </p>
+<p class="answer">Spark can either spill it to disk or recompute the partitions that don't fit in RAM each time they are requested. By default, it uses recomputation, but you can set a dataset's <a href="/docs/latest/scala-programming-guide.html#rdd-persistence">storage level</a> to <code>MEMORY_AND_DISK</code> to avoid this. </p>
<p class="question">How can I run Spark on a cluster?</p>
-<p class="answer">You can use either the <a href="/docs/latest/spark-standalone.html">standalone deploy mode</a>, which depends only on Java, or the <a href="/docs/latest/running-on-mesos.html">Apache Mesos</a> cluster manager.</p>
-<p>Note that you can also run Spark locally (possibly on multiple cores) without any special setup by just passing <tt>local[N]</tt> as the master URL, where <tt>N</tt> is the number of parallel threads you want.</p>
+<p class="answer">You can use either the <a href="/docs/latest/spark-standalone.html">standalone deploy mode</a>, which only needs Java to be installed on each node, or the <a href="/docs/latest/running-on-mesos.html">Mesos</a> and <a href="/docs/latest/running-on-yarn.html">YARN</a> cluster managers. If you'd like to run on Amazon EC2, Spark provides <a href="/docs/latest/ec2-scripts.html}}">EC2 scripts</a> to automatically launch a cluster.</p>
+
+<p>Note that you can also run Spark locally (possibly on multiple cores) without any special setup by just passing <code>local[N]</code> as the master URL, where <code>N</code> is the number of parallel threads you want.</p>
-<p class="question">I don't know Scala; how hard is it to pick it up to use Spark?</p>
-<p class="answer">Scala itself is pretty easy to pick up if you have Java experience. Check out <a href="http://www.artima.com/scalazine/articles/steps.html">First Steps to Scala</a> for a quick introduction, the <a href="http://www.scala-lang.org/docu/files/ScalaTutorial.pdf">Scala tutorial for Java programmers</a>, or the free online book <a href="http://www.artima.com/pins1ed/">Programming in Scala</a>.</p>
-<p>Spark 0.6 also added a <a href="/docs/latest/java-programming-guide.html">Java API</a>, letting you use Spark from Java, and Spark 0.7 added a <a href="/docs/latest/python-programming-guide.html">Python API</a>.</p>
+<p class="question">Do I need Hadoop to run Spark?</p>
+<p class="answer">No, but if you run on a cluster, you will need some form of shared file system (for example, NFS mounted at the same path on each node). If you have this type of filesystem, you can just deploy Spark in standalone mode.</p>
+
+<p class="question">How can I access data in S3?</p>
+<p class="answer">Use the <code>s3n://</code> URI scheme (<code>s3n://bucket/path</code>). You will also need to set your Amazon security credentials, either by setting the environment variables <code>AWS_ACCESS_KEY_ID</code> and <code>AWS_SECRET_ACCESS_KEY</code> before your program runs, or by setting <code>fs.s3.awsAccessKeyId</code> and <code>fs.s3.awsSecretAccessKey</code> in <code>SparkContext.hadoopConfiguration</code>.</p>
+
+<p class="question">What are good resources for learning Scala?</p>
+<p class="answer">Check out <a href="http://www.artima.com/scalazine/articles/steps.html">First Steps to Scala</a> for a quick introduction, the <a href="http://www.scala-lang.org/docu/files/ScalaTutorial.pdf">Scala tutorial for Java programmers</a>, or the free online book <a href="http://www.artima.com/pins1ed/">Programming in Scala</a>. Scala is easy to transition to if you have Java experience or experience in a similarly high-level language (e.g. Ruby).</p>
+
+<p>In addition, Spark also has <a href="/docs/latest/java-programming-guide.html">Java</a> and <a href="/docs/latest/python-programming-guide.html">Python</a> APIs.</p>
<p class="question">What license is Spark under?</p>
<p class="answer">Starting in version 0.8, Spark is under the <a href="http://www.apache.org/licenses/LICENSE-2.0.html">Apache 2.0 license</a>. Previous versions used the <a href="https://github.com/mesos/spark/blob/branch-0.7/LICENSE">BSD license</a>.</p>
<p class="question">How can I contribute to Spark?</p>
-<p class="answer">Contact the <a href="http://groups.google.com/group/spark-users">mailing list</a> or send us a pull request on GitHub. We're glad to hear about your experience using Spark and to accept patches </p>
+<p class="answer">Contact the <a href="/community.html">mailing list</a> or send us a pull request on <a href="https://github.com/apache/incubator-spark">GitHub</a> (instructions <a href="https://cwiki.apache.org/confluence/display/SPARK/Contributing+to+Spark">here</a>). We're glad to hear about your experience using Spark and to accept patches.</p>
<p>If you would like to report an issue, post it to the <a href="https://spark-project.atlassian.net/browse/SPARK">Spark issue tracker</a>.</p>
<p class="question">Where can I get more help?</p>
-<p class="answer">Please post on the <a href="http://groups.google.com/group/spark-users">spark-users</a> mailing list. We'll be glad to help!</p>
+<p class="answer">Please post on the <a href="http://apache-spark-user-list.1001560.n3.nabble.com">Spark Users</a> mailing list. We'll be glad to help!</p>
- </article><!-- #post -->
-
- </div><!-- #content -->
-
- <footer id="colophon" role="contentinfo">
- <div id="site-generator">
- <p style="padding-top: 0; padding-bottom: 15px;">
- Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
- <a href="http://incubator.apache.org/" style="border: none;">
- <img style="vertical-align: middle; border: none;" src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
- </a>
- </p>
</div>
-</footer><!-- #colophon -->
+</div>
+
+
+
+<footer class="small">
+ <hr>
+ Apache Spark is an effort undergoing incubation at The Apache Software Foundation.
+ <a href="http://incubator.apache.org/" style="border: none;">
+ <img style="vertical-align: middle; float: right; margin-bottom: 15px;"
+ src="/images/incubator-logo.png" alt="Apache Incubator" title="Apache Incubator" />
+ </a>
+</footer>
- </div><!-- #primary -->
- </div><!-- #main -->
-</div><!-- #page -->
+</div>
+<script src="https://code.jquery.com/jquery.js"></script>
+<script src="//netdna.bootstrapcdn.com/bootstrap/3.0.3/js/bootstrap.min.js"></script>
+<script src="/js/lang-tabs.js"></script>
</body>
</html>
Added: incubator/spark/site/images/hadoop.jpg
URL: http://svn.apache.org/viewvc/incubator/spark/site/images/hadoop.jpg?rev=1560502&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/spark/site/images/hadoop.jpg
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/spark/site/images/logistic-regression.png
URL: http://svn.apache.org/viewvc/incubator/spark/site/images/logistic-regression.png?rev=1560502&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/spark/site/images/logistic-regression.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/spark/site/images/logistic-regression2.png
URL: http://svn.apache.org/viewvc/incubator/spark/site/images/logistic-regression2.png?rev=1560502&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/spark/site/images/logistic-regression2.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/spark/site/images/scaling.png
URL: http://svn.apache.org/viewvc/incubator/spark/site/images/scaling.png?rev=1560502&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/spark/site/images/scaling.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: incubator/spark/site/images/spark-logo.png
URL: http://svn.apache.org/viewvc/incubator/spark/site/images/spark-logo.png?rev=1560502&view=auto
==============================================================================
Binary file - no diff available.
Propchange: incubator/spark/site/images/spark-logo.png
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream