You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@crail.apache.org by pe...@apache.org on 2018/08/14 13:50:47 UTC
[1/3] incubator-crail-website git commit: Publishing from
ff498052418181475486b63a7d66169591496498
Repository: incubator-crail-website
Updated Branches:
refs/heads/asf-site 59f881c80 -> 5b208cb94
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/page5/index.html
----------------------------------------------------------------------
diff --git a/content/blog/page5/index.html b/content/blog/page5/index.html
index e4f4121..2aaeb13 100644
--- a/content/blog/page5/index.html
+++ b/content/blog/page5/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Latest posts</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/page5-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/page5-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/page5-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/page5-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -167,6 +167,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/community/index.html
----------------------------------------------------------------------
diff --git a/content/community/index.html b/content/community/index.html
index 94beb0b..9aa4a89 100644
--- a/content/community/index.html
+++ b/content/community/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Community</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/community-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/community-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/community-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/community-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li class="active">
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -112,6 +112,6 @@ For any potential changes/proposals we recommend that you open a JIRA ticket to
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/documentation/index.html
----------------------------------------------------------------------
diff --git a/content/documentation/index.html b/content/documentation/index.html
index 331a35b..ee10248 100644
--- a/content/documentation/index.html
+++ b/content/documentation/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Documentation</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/documentation-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/documentation-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/documentation-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/documentation-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li class="active">
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -360,6 +360,6 @@ you propose, and let us know.</p>
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/download/index.html
----------------------------------------------------------------------
diff --git a/content/download/index.html b/content/download/index.html
index bc992a2..220fac2 100644
--- a/content/download/index.html
+++ b/content/download/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Downloads</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/download-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/download-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/download-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/download-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li class="active">
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -121,6 +121,6 @@ For security, hash and signature files are always hosted at Apache.</p>
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/feed.xml
----------------------------------------------------------------------
diff --git a/content/feed.xml b/content/feed.xml
index 3f03ab5..5a18596 100644
--- a/content/feed.xml
+++ b/content/feed.xml
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.7.0">Jekyll</generator><link href="http://crail.incubator.apache.org//feed.xml" rel="self" type="application/atom+xml" /><link href="http://crail.incubator.apache.org//" rel="alternate" type="text/html" /><updated>2018-08-14T13:26:12+02:00</updated><id>http://crail.incubator.apache.org//</id><title type="html">The Apache Crail (Incubating) Project</title><entry><title type="html">Sql P1 News</title><link href="http://crail.incubator.apache.org//blog/2018/08/sql-p1-news.html" rel="alternate" type="text/html" title="Sql P1 News" /><published>2018-08-09T00:00:00+02:00</published><updated>2018-08-09T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/08/sql-p1-news</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/08/sql-p1-news.html"><p>A new blog <a href="http://crail.incubator.apache.org/blo
g/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p></content><author><name></name></author><category term="news" /><summary type="html">A new blog post discussing file formats performance is now online</summary></entry><entry><title type="html">SQL Performance: Part 1 - Input File Formats</title><link href="http://crail.incubator.apache.org//blog/2018/08/sql-p1.html" rel="alternate" type="text/html" title="SQL Performance: Part 1 - Input File Formats" /><published>2018-08-08T00:00:00+02:00</published><updated>2018-08-08T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/08/sql-p1</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/08/sql-p1.html"><div style="text-align: justify">
+<?xml version="1.0" encoding="utf-8"?><feed xmlns="http://www.w3.org/2005/Atom" ><generator uri="https://jekyllrb.com/" version="3.6.2">Jekyll</generator><link href="http://crail.incubator.apache.org//feed.xml" rel="self" type="application/atom+xml" /><link href="http://crail.incubator.apache.org//" rel="alternate" type="text/html" /><updated>2018-08-14T15:50:25+02:00</updated><id>http://crail.incubator.apache.org//</id><title type="html">The Apache Crail (Incubating) Project</title><entry><title type="html">Sql P1 News</title><link href="http://crail.incubator.apache.org//blog/2018/08/sql-p1-news.html" rel="alternate" type="text/html" title="Sql P1 News" /><published>2018-08-09T00:00:00+02:00</published><updated>2018-08-09T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/08/sql-p1-news</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/08/sql-p1-news.html"><p>A new blog <a href="//crail.incubator.apache.org/blog/201
8/08/sql-p1.html">post</a> discussing file formats performance is now online</p></content><author><name></name></author><category term="news" /><summary type="html">A new blog post discussing file formats performance is now online</summary></entry><entry><title type="html">SQL Performance: Part 1 - Input File Formats</title><link href="http://crail.incubator.apache.org//blog/2018/08/sql-p1.html" rel="alternate" type="text/html" title="SQL Performance: Part 1 - Input File Formats" /><published>2018-08-08T00:00:00+02:00</published><updated>2018-08-08T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/08/sql-p1</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/08/sql-p1.html"><div style="text-align: justify">
<p>
This is the first blog post in a multi-part series where we will focus on relational data processing performance (e.g., SQL) in presence of high-performance network and storage devices - the kind of devices that Crail targets. Relational data processing is one of the most popular and versatile workloads people run in the cloud. The general idea is that data is stored in tables with a schema, and is processed using a domain specific language like SQL. Examples of some popular systems that support such relational data analytics in the cloud are <a href="https://spark.apache.org/sql/">Apache Spark/SQL</a>, <a href="https://hive.apache.org/">Apache Hive</a>, <a href="https://impala.apache.org/">Apache Impala</a>, etc. In this post, we discuss the important first step in relational data processing, which is the reading of input data tables.
</p>
@@ -35,11 +35,11 @@ This is the first blog post in a multi-part series where we will focus on relati
<p>In a typical cloud-based relational data processing setup, the input data is stored on an external data storage solution like HDFS or AWS S3. Data tables and their associated schema are converted into a storage-friendly format for optimal performance. Examples of some popular and familiar file formats are <a href="https://parquet.apache.org/">Apache Parquet</a>, <a href="https://orc.apache.org/">Apache ORC</a>, <a href="https://avro.apache.org/">Apache Avro</a>, <a href="https://en.wikipedia.org/wiki/JSON">JSON</a>, etc. More recently, <a href="https://arrow.apache.org/">Apache Arrow</a> has been introduced to standardize the in-memory columnar data representation between multiple frameworks. There is no one size fits all as all these formats have their own strengths, weaknesses, and features. In this blog, we are specifically interested in the performance of these
formats on modern high-performance networking and storage devices.</p>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/outline.svg" width="550" /><figcaption>Figure 1: The benchmarking setup with HDFS and file formats on a 100 Gbps network with NVMe flash devices. All formats contains routines for compression, encoding, and value materialization with associated I/O buffer management and data copies routines.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/outline.svg" width="550" /><figcaption>Figure 1: The benchmarking setup with HDFS and file formats on a 100 Gbps network with NVMe flash devices. All formats contains routines for compression, encoding, and value materialization with associated I/O buffer management and data copies routines.<p></p></figcaption></div></figure>
<p>To benchmark the performance of file formats, we wrote a set of micro-benchmarks which are available at <a href="https://github.com/zrlio/fileformat-benchmarks">https://github.com/zrlio/fileformat-benchmarks</a>. We cannot use typical SQL micro-benchmarks because every SQL engine has its own favorite file format, on which it performs the best. Hence, in order to ensure parity, we decoupled the performance of reading the input file format from the SQL query processing by writing simple table reading micro-benchmarks. Our benchmark reads in the store_sales table from the TPC-DS dataset (scale factor 100), and calculates a sum of values present in the table. The table contains 23 columns of integers, doubles, and longs.</p>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/performance-all.svg" width="550" /><figcaption>Figure 2: Performance of JSON, Avro, Parquet, ORC, and Arrow on NVMe devices over a 100 Gbps network.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/performance-all.svg" width="550" /><figcaption>Figure 2: Performance of JSON, Avro, Parquet, ORC, and Arrow on NVMe devices over a 100 Gbps network.<p></p></figcaption></div></figure>
<p>We evaluate the performance of the benchmark on a 3 node HDFS cluster connected using 100 Gbps RoCE. One datanode in HDFS contains 4 NVMe devices with a collective aggregate bandwidth of 12.5 GB/sec (equals to 100 Gbps, hence, we have a balanced network and storage performance). Figure 2 shows our results where none of the file formats is able to deliver the full hardware performance for reading input files. One third of the performance is already lost in HDFS (maximum throughput 74.9 Gbps out of possible 100 Gbps). The rest of the performance is lost inside the file format implementation, which needs to deal with encoding, buffer and I/O management, compression, etc. The best performer is Apache Arrow which is designed for in-memory columnar datasets. The performance of these file formats are bounded by the performance of the CPU, which is 100% loaded during the experiment. For a detailed analysis of the file formats, please refer to our paper - <a href="https://ww
w.usenix.org/conference/atc18/presentation/trivedi">Albis: High-Performance File Format for Big Data Systems (USENIX, ATC’18)</a>.</p>
@@ -53,7 +53,7 @@ This is the first blog post in a multi-part series where we will focus on relati
<li>Careful object materialization using a binary API: To optimize the runtime representation in managed runtimes like the JVM, only objects which are necessary for SQL processing are materialized. Otherwise, a 4 byte integer can be passed around as a byte array (using the binary API of Albis).</li>
</ul>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/core-scalability.svg" width="550" /><figcaption>Figure 3: Core scalability of JSON, Avro, Parquet, ORC, Arrow, and Albis on HDFS/NVMe.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/core-scalability.svg" width="550" /><figcaption>Figure 3: Core scalability of JSON, Avro, Parquet, ORC, Arrow, and Albis on HDFS/NVMe.<p></p></figcaption></div></figure>
<p>Using the Albis format, we revise our previous experiment where we read the input store_sales table from HDFS. In the figure above, we show the performance of Albis and other file formats with number of CPU cores involved. At the right hand of the x-axis, we have performance with all 16 cores engaged, hence, representing the peak possible performance. As evident, Albis delivers 59.9 Gbps out of 74.9 Gbps possible bandwidth with HDFS over NVMe. Albis performance is 1.9 - 21.4x better than other file formats. To give an impression where the performance is coming from, in the table below we show some micro-architectural features for Parquet, ORC, Arrow, and Albis. Our previously discussed design ideas in Albis result in a shorter code path (shown as less instructions required for each row), better cache performance (shows as lower cache misses per row), and clearly better performance (shown as nanoseconds required per row for processing). For a detailed evaluation of Albis ple
ase refer to our paper.</p>
@@ -92,9 +92,9 @@ This is the first blog post in a multi-part series where we will focus on relati
<h3 id="apache-crail-incubating-with-albis">Apache Crail (Incubating) with Albis</h3>
-<p>For our final experiment, we try to answer the question what it would take to deliver the full 100 Gbps bandwidth for Albis. Certainly, the first bottleneck is to improve the base storage layer performance. Here we use Apache Crail (Incubating) with its <a href="https://en.wikipedia.org/wiki/NVM_Express#NVMeOF">NVMeF</a> storage tier. This tier uses <a href="https://github.com/zrlio/jNVMf">jNVMf library</a> to implement NVMeF stack in Java. As we have shown in a previous blog <a href="http://crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">post</a> that Crail’s NVMeF tier can deliver performance (97.8 Gbps) very close to the hardware limits. Hence, Albis with Crail is a perfect setup to evaluate on high-performance NVMe and RDMA devices. Before we get there, let’s get some calculations right. The store_sales table in the TPC-DS dataset has a data density of 93.9% (out of 100 bytes, only
93.9 is data, others are null values). As we measure the goodput, the expected performance of Albis on Crail is 93.9% of 97.8 Gbps, which calculates to 91.8 Gbps. In our experiments, Albis on Crail delivers 85.5 Gbps. Figure 4 shows more detailed results.</p>
+<p>For our final experiment, we try to answer the question what it would take to deliver the full 100 Gbps bandwidth for Albis. Certainly, the first bottleneck is to improve the base storage layer performance. Here we use Apache Crail (Incubating) with its <a href="https://en.wikipedia.org/wiki/NVM_Express#NVMeOF">NVMeF</a> storage tier. This tier uses <a href="https://github.com/zrlio/jNVMf">jNVMf library</a> to implement NVMeF stack in Java. As we have shown in a previous blog <a href="//crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">post</a> that Crail’s NVMeF tier can deliver performance (97.8 Gbps) very close to the hardware limits. Hence, Albis with Crail is a perfect setup to evaluate on high-performance NVMe and RDMA devices. Before we get there, let’s get some calculations right. The store_sales table in the TPC-DS dataset has a data density of 93.9% (out of 100 bytes, only 93.9
is data, others are null values). As we measure the goodput, the expected performance of Albis on Crail is 93.9% of 97.8 Gbps, which calculates to 91.8 Gbps. In our experiments, Albis on Crail delivers 85.5 Gbps. Figure 4 shows more detailed results.</p>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/albis-crail.svg" width="550" /><figcaption>Figure 4: Performance of Albis on Crail.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/albis-crail.svg" width="550" /><figcaption>Figure 4: Performance of Albis on Crail.<p></p></figcaption></div></figure>
<p>The left half of the figure shows the performance scalability of Albis on Crail in a setup with 1 core (8.9 Gbps) to 16 cores (85.5 Gbps). In comparison, the right half of the figure shows the performance of Crail on HDFS/NVMe at 59.9 Gbps, and on Crail/NVMe at 85.5 Gbps. The last bar shows the performance of Albis if the benchmark does not materialize Java object values. In this configuration, Albis on Crail delivers 91.3 Gbps, which is very close to the expected peak of 91.8 Gbps.</p>
@@ -107,7 +107,7 @@ In this first blog of a multipart series, we have looked at the data ingestion p
<p>
Stay tuned for the next part where we look at workload-level performance in Spark/SQL on modern high-performance networking and storage devices. Meanwhile let us know if you have any feedback or comments.
</p>
-</div></content><author><name>Animesh Trivedi, Patrick Stuedi, Jonas Pfefferle, Adrian Schuepbach, and Bernard Metzler</name></author><category term="blog" /><summary type="html">This is the first blog post in a multi-part series where we will focus on relational data processing performance (e.g., SQL) in presence of high-performance network and storage devices - the kind of devices that Crail targets. Relational data processing is one of the most popular and versatile workloads people run in the cloud. The general idea is that data is stored in tables with a schema, and is processed using a domain specific language like SQL. Examples of some popular systems that support such relational data analytics in the cloud are Apache Spark/SQL, Apache Hive, Apache Impala, etc. In this post, we discuss the important first step in relational data processing, which is the reading of input data tables.</summary></entry><entry><title type="html">Sparksummit</title><link href="http://crail.i
ncubator.apache.org//blog/2018/06/sparksummit.html" rel="alternate" type="text/html" title="Sparksummit" /><published>2018-06-05T00:00:00+02:00</published><updated>2018-06-05T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/sparksummit</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/sparksummit.html"><p>A Spark serverless architecture powered by Crail will be presented today at the <a href="https://databricks.com/session/serverless-machine-learning-on-modern-hardware-using-apache-spark">Spark Summit</a></p></content><author><name></name></author><category term="news" /><summary type="html">A Spark serverless architecture powered by Crail will be presented today at the Spark Summit</summary></entry><entry><title type="html">Dataworks</title><link href="http://crail.incubator.apache.org//blog/2018/06/dataworks.html" rel="alternate" type="text/html" title="Dataworks" /><published>2018-06-05T
00:00:00+02:00</published><updated>2018-06-05T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/dataworks</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/dataworks.html"><p>Apache Crail (incubating) to feature in the <a href="https://dataworkssummit.com/san-jose-2018/session/data-processing-at-the-speed-of-100-gbpsapache-crail-incubating/">DataWorks Summit</a> on June 21st</p></content><author><name></name></author><category term="news" /><summary type="html">Apache Crail (incubating) to feature in the DataWorks Summit on June 21st</summary></entry><entry><title type="html">Apache Release</title><link href="http://crail.incubator.apache.org//blog/2018/06/apache-release.html" rel="alternate" type="text/html" title="Apache Release" /><published>2018-06-04T00:00:00+02:00</published><updated>2018-06-04T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/apache-release</
id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/apache-release.html"><p>Apache Crail 1.0 incubator <a href="http://crail.incubator.apache.org/download">release</a></p></content><author><name></name></author><category term="news" /><summary type="html">Apache Crail 1.0 incubator release</summary></entry><entry><title type="html">Apache</title><link href="http://crail.incubator.apache.org//blog/2018/01/apache.html" rel="alternate" type="text/html" title="Apache" /><published>2018-01-22T00:00:00+01:00</published><updated>2018-01-22T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2018/01/apache</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/01/apache.html"><p>Crail is now an Apache Incubator project!</p></content><author><name></name></author><category term="news" /><summary type="html">Crail is now an Apache Incubator project!</summary></entry><entry><tit
le type="html">Iops</title><link href="http://crail.incubator.apache.org//blog/2017/11/iops.html" rel="alternate" type="text/html" title="Iops" /><published>2017-11-23T00:00:00+01:00</published><updated>2017-11-23T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/iops</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/iops.html"><p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crail’s metadata performance and scalability</p></content><author><name></name></author><category term="news" /><summary type="html">New blog post about Crail’s metadata performance and scalability</summary></entry><entry><title type="html">Crail Storage Performance – Part III: Metadata</title><link href="http://crail.incubator.apache.org//blog/2017/11/crail-metadata.html" rel="alternate" type="text/html" title="Crail Storage Performance -- Part III: Metadata"
/><published>2017-11-21T00:00:00+01:00</published><updated>2017-11-21T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/crail-metadata</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/crail-metadata.html"><div style="text-align: justify">
+</div></content><author><name>Animesh Trivedi, Patrick Stuedi, Jonas Pfefferle, Adrian Schuepbach, and Bernard Metzler</name></author><category term="blog" /><summary type="html">This is the first blog post in a multi-part series where we will focus on relational data processing performance (e.g., SQL) in presence of high-performance network and storage devices - the kind of devices that Crail targets. Relational data processing is one of the most popular and versatile workloads people run in the cloud. The general idea is that data is stored in tables with a schema, and is processed using a domain specific language like SQL. Examples of some popular systems that support such relational data analytics in the cloud are Apache Spark/SQL, Apache Hive, Apache Impala, etc. In this post, we discuss the important first step in relational data processing, which is the reading of input data tables.</summary></entry><entry><title type="html">Sparksummit</title><link href="http://crail.i
ncubator.apache.org//blog/2018/06/sparksummit.html" rel="alternate" type="text/html" title="Sparksummit" /><published>2018-06-05T00:00:00+02:00</published><updated>2018-06-05T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/sparksummit</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/sparksummit.html"><p>A Spark serverless architecture powered by Crail will be presented today at the <a href="https://databricks.com/session/serverless-machine-learning-on-modern-hardware-using-apache-spark">Spark Summit</a></p></content><author><name></name></author><category term="news" /><summary type="html">A Spark serverless architecture powered by Crail will be presented today at the Spark Summit</summary></entry><entry><title type="html">Dataworks</title><link href="http://crail.incubator.apache.org//blog/2018/06/dataworks.html" rel="alternate" type="text/html" title="Dataworks" /><published>2018-06-05T
00:00:00+02:00</published><updated>2018-06-05T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/dataworks</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/dataworks.html"><p>Apache Crail (incubating) to feature in the <a href="https://dataworkssummit.com/san-jose-2018/session/data-processing-at-the-speed-of-100-gbpsapache-crail-incubating/">DataWorks Summit</a> on June 21st</p></content><author><name></name></author><category term="news" /><summary type="html">Apache Crail (incubating) to feature in the DataWorks Summit on June 21st</summary></entry><entry><title type="html">Apache Release</title><link href="http://crail.incubator.apache.org//blog/2018/06/apache-release.html" rel="alternate" type="text/html" title="Apache Release" /><published>2018-06-04T00:00:00+02:00</published><updated>2018-06-04T00:00:00+02:00</updated><id>http://crail.incubator.apache.org//blog/2018/06/apache-release</
id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/06/apache-release.html"><p>Apache Crail 1.0 incubator <a href="//crail.incubator.apache.org/download">release</a></p></content><author><name></name></author><category term="news" /><summary type="html">Apache Crail 1.0 incubator release</summary></entry><entry><title type="html">Apache</title><link href="http://crail.incubator.apache.org//blog/2018/01/apache.html" rel="alternate" type="text/html" title="Apache" /><published>2018-01-22T00:00:00+01:00</published><updated>2018-01-22T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2018/01/apache</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2018/01/apache.html"><p>Crail is now an Apache Incubator project!</p></content><author><name></name></author><category term="news" /><summary type="html">Crail is now an Apache Incubator project!</summary></entry><entry><title ty
pe="html">Iops</title><link href="http://crail.incubator.apache.org//blog/2017/11/iops.html" rel="alternate" type="text/html" title="Iops" /><published>2017-11-23T00:00:00+01:00</published><updated>2017-11-23T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/iops</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/iops.html"><p>New blog <a href="//crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crail’s metadata performance and scalability</p></content><author><name></name></author><category term="news" /><summary type="html">New blog post about Crail’s metadata performance and scalability</summary></entry><entry><title type="html">Crail Storage Performance – Part III: Metadata</title><link href="http://crail.incubator.apache.org//blog/2017/11/crail-metadata.html" rel="alternate" type="text/html" title="Crail Storage Performance -- Part III: Metadata" /><publis
hed>2017-11-21T00:00:00+01:00</published><updated>2017-11-21T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/crail-metadata</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/crail-metadata.html"><div style="text-align: justify">
<p>
This is part III of our series of posts discussing Crail's raw storage performance. This part is about Crail's metadata performance and scalability.
</p>
@@ -142,11 +142,11 @@ This is part III of our series of posts discussing Crail's raw storage performan
<div style="text-align: justify">
<p>
-As described in <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">part I</a>, Crail data operations are composed of actual data transfers and metadata operations. Examples of metadata operations are operations for creating or modifying the state of a file, or operations to lookup the storage server that stores a particular range (block) of a file. In Crail, all the metadata is managed by the namenode(s) (as opposed to the data which is managed by the storage nodes). Clients interact with Crail namenodes via Remote Procedure Calls (RPCs). Crail supports multiple RPC protocols for different types of networks and also offers a pluggable RPC interface so that new RPC bindings can be implemented easily. On RDMA networks, the default DaRPC (<a href="https://dl.acm.org/citation.cfm?id=2670994">DaRPC paper</a>, <a href="http://github.com/zrlio/darpc">DaRPC GitHub</a>) based RPC binding provides the be
st performance. The figure below gives an overview of the Crail metadata processing in a DaRPC configuration.
+As described in <a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">part I</a>, Crail data operations are composed of actual data transfers and metadata operations. Examples of metadata operations are operations for creating or modifying the state of a file, or operations to lookup the storage server that stores a particular range (block) of a file. In Crail, all the metadata is managed by the namenode(s) (as opposed to the data which is managed by the storage nodes). Clients interact with Crail namenodes via Remote Procedure Calls (RPCs). Crail supports multiple RPC protocols for different types of networks and also offers a pluggable RPC interface so that new RPC bindings can be implemented easily. On RDMA networks, the default DaRPC (<a href="https://dl.acm.org/citation.cfm?id=2670994">DaRPC paper</a>, <a href="http://github.com/zrlio/darpc">DaRPC GitHub</a>) based RPC binding provides the best pe
rformance. The figure below gives an overview of the Crail metadata processing in a DaRPC configuration.
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/rpc.png" width="480" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/rpc.png" width="480" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -159,10 +159,10 @@ Crail supports partitioning of metadata across several namenods. Thereby, metada
<div style="text-align: justify">
<p>
-In two of the previous blogs (<a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>, <a href="http://crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">NVMf</a>) we have already shown that Crail metadata operations are very low latency. Essentially a single metadata operation issued by a remote client takes 5-6 microseconds, which is only slightly more than the raw network latency of the RDMA network fabric. In this blog, we want to explore the scalability of Crail's metadata management, that is, the number of clients Crail can support, or how Crail scales as the cluster size increases. The level of scalability of Crail is mainly determined by the number of metadata operations Crail can process concurrently, a metric that is often referred to as IOPS. The higher the number of IOPS the system can handle, the more clients can concurrently use Crail without performance loss.
+In two of the previous blogs (<a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>, <a href="//crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">NVMf</a>) we have already shown that Crail metadata operations are very low latency. Essentially a single metadata operation issued by a remote client takes 5-6 microseconds, which is only slightly more than the raw network latency of the RDMA network fabric. In this blog, we want to explore the scalability of Crail's metadata management, that is, the number of clients Crail can support, or how Crail scales as the cluster size increases. The level of scalability of Crail is mainly determined by the number of metadata operations Crail can process concurrently, a metric that is often referred to as IOPS. The higher the number of IOPS the system can handle, the more clients can concurrently use Crail without performance loss.
</p>
<p>
-An important metadata operation is ''getFile()'', which is used by clients to lookup the status of a file (whether the file exists, what size it has, etc.). The ''getFile()'' operation is served by Crail's fast lock-free map and in spirit is very similar to the ''getBlock()'' metadata operation (used by clients to query which storage nodes holds a particular block). In a typical Crail use case, ''getFile()'' and ''getBlock()'' are responsible for the peak metadata load at a namenode. In this experiment, we measure the achievable IOPS on the server side in an artificial configuration with many clients distributed across the cluster issuing ''getFile()'' in a tight loop. Note that the client side RPC interface in Crail is asynchronous, thus, clients can issue multiple metadata operations without blocking while asynchronously waiting for the result. In the experiments below, each client may have a maximum of 128 ''getFile()'' operations outstanding at any point in time. In a practical
scenario, Crail clients may also have multiple metadata operations in flight either because clients are shared by different cores, or because Crail interleaves metadata and data operations (see <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>). What makes the benchmark artificial is that clients exclusively focus on generating load for the namenode and thereby are neither performing data operations nor are they doing any compute. The basic command of the benchmark as executed by each of the individual clients is given by the following command:
+An important metadata operation is ''getFile()'', which is used by clients to lookup the status of a file (whether the file exists, what size it has, etc.). The ''getFile()'' operation is served by Crail's fast lock-free map and in spirit is very similar to the ''getBlock()'' metadata operation (used by clients to query which storage nodes holds a particular block). In a typical Crail use case, ''getFile()'' and ''getBlock()'' are responsible for the peak metadata load at a namenode. In this experiment, we measure the achievable IOPS on the server side in an artificial configuration with many clients distributed across the cluster issuing ''getFile()'' in a tight loop. Note that the client side RPC interface in Crail is asynchronous, thus, clients can issue multiple metadata operations without blocking while asynchronously waiting for the result. In the experiments below, each client may have a maximum of 128 ''getFile()'' operations outstanding at any point in time. In a practical
scenario, Crail clients may also have multiple metadata operations in flight either because clients are shared by different cores, or because Crail interleaves metadata and data operations (see <a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>). What makes the benchmark artificial is that clients exclusively focus on generating load for the namenode and thereby are neither performing data operations nor are they doing any compute. The basic command of the benchmark as executed by each of the individual clients is given by the following command:
</p>
</div>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t getMultiFileAsync -f / -k 10000000 -b 128
@@ -205,7 +205,7 @@ The line of the raw number of IOPS, labeled ''ib send'' is shown in the same gra
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
<p>
@@ -222,7 +222,7 @@ To increase the number of IOPS the overall system can handle, we allow starting
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -250,7 +250,7 @@ namenodes happening, which should lead to linear scalability.
<div style="text-align: justify">
<p>
Let us look at a concrete application, which ideally runs on a large cluster:
-TeraSort. In a previous blog, <a href="http://crail.incubator.apache.org/blog/2017/01/sorting.html">sorting</a>,
+TeraSort. In a previous blog, <a href="//crail.incubator.apache.org/blog/2017/01/sorting.html">sorting</a>,
we analyze performance characteristics of TeraSort on Crail on a big cluster
of 128 nodes, where we run 384 executors in total. This already proves that
Crail can at least handle 384 clients. Now we analyze the theoretical number
@@ -284,7 +284,7 @@ namenode over the elapsed runtime of the TeraSort application.
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -448,7 +448,7 @@ plot shows the number of IOPS relative to the number of clients.
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -502,7 +502,7 @@ the blog with the latest numbers as soon as the bug is fixed.
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -533,7 +533,7 @@ of 30Mio/s with 4 namenodes).
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -553,4 +553,4 @@ of operations even compared to a C++-based system like RAMCloud.
<p>
In this blog we show three key points of Crail: First, Crail's namenode performs the same as ib_send_bw with realistic parameters in terms of IOPS. This shows that the actual processing of the RPC is implemented efficiently. Second, with only one namenode, Crail performs 10x to 50x better than RAMCloud and HDFS, two popular systems, where RAMCloud is RDMA-based and implemented natively. Third, Crail's metadata service can be scaled out to serve large number of clients. We have shown that Crail offers near linear scaling with up to 4 namenodes, offering a performance that is sufficient to serve several 1000s of clients.
</p>
-</div></content><author><name>Adrian Schuepbach and Patrick Stuedi</name></author><category term="blog" /><summary type="html">This is part III of our series of posts discussing Crail's raw storage performance. This part is about Crail's metadata performance and scalability.</summary></entry><entry><title type="html">Floss</title><link href="http://crail.incubator.apache.org//blog/2017/11/floss.html" rel="alternate" type="text/html" title="Floss" /><published>2017-11-17T00:00:00+01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/floss</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/floss.html"><p>Crail features in the <a href="https://twit.tv/shows/floss-weekly/episodes/458?autostart=false">FLOSS weekly podcast</a></p></content><author><name></name></author><category term="news" /><summary type="html">Crail features in the FLOSS weekly podcast</sum
mary></entry><entry><title type="html">Blog</title><link href="http://crail.incubator.apache.org//blog/2017/11/blog.html" rel="alternate" type="text/html" title="Blog" /><published>2017-11-17T00:00:00+01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/blog</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/blog.html"><p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p></content><author><name></name></author><category term="news" /><summary type="html">New blog post about SparkRDMA and Crail shuffle plugins</summary></entry></feed>
\ No newline at end of file
+</div></content><author><name>Adrian Schuepbach and Patrick Stuedi</name></author><category term="blog" /><summary type="html">This is part III of our series of posts discussing Crail's raw storage performance. This part is about Crail's metadata performance and scalability.</summary></entry><entry><title type="html">Floss</title><link href="http://crail.incubator.apache.org//blog/2017/11/floss.html" rel="alternate" type="text/html" title="Floss" /><published>2017-11-17T00:00:00+01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/floss</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/floss.html"><p>Crail features in the <a href="https://twit.tv/shows/floss-weekly/episodes/458?autostart=false">FLOSS weekly podcast</a></p></content><author><name></name></author><category term="news" /><summary type="html">Crail features in the FLOSS weekly podcast</sum
mary></entry><entry><title type="html">Blog</title><link href="http://crail.incubator.apache.org//blog/2017/11/blog.html" rel="alternate" type="text/html" title="Blog" /><published>2017-11-17T00:00:00+01:00</published><updated>2017-11-17T00:00:00+01:00</updated><id>http://crail.incubator.apache.org//blog/2017/11/blog</id><content type="html" xml:base="http://crail.incubator.apache.org//blog/2017/11/blog.html"><p>New blog <a href="//crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p></content><author><name></name></author><category term="news" /><summary type="html">New blog post about SparkRDMA and Crail shuffle plugins</summary></entry></feed>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/index.html
----------------------------------------------------------------------
diff --git a/content/index.html b/content/index.html
index c45a1a5..6e2774c 100644
--- a/content/index.html
+++ b/content/index.html
@@ -4,12 +4,12 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Overview</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/crail-stack-preview.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/crail-stack-preview.png" />
+ href="//crail.incubator.apache.org/blog/blog.xml">
+ <meta property="og:image" content="//crail.incubator.apache.org/img/crail-stack-preview.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/crail-stack-preview.png" />
</head>
<body>
@@ -20,50 +20,50 @@
<li class="active">
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -79,7 +79,7 @@
<br>
<div>
<center>
-<img src="http://crail.incubator.apache.org/img/crail_stack3.png" align="middle" style="width: 340px; text-align:center;">
+<img src="//crail.incubator.apache.org/img/crail_stack3.png" align="middle" style="width: 340px; text-align:center;">
</center>
</div>
<br>
@@ -109,7 +109,7 @@
<li class="shortnews">
<span class="date">August 9, 2018</span>
- <p>A new blog <a href="http://crail.incubator.apache.org/blog/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p>
+ <p>A new blog <a href="//crail.incubator.apache.org/blog/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p>
</li>
@@ -133,7 +133,7 @@
<li class="shortnews">
<span class="date">June 4, 2018</span>
- <p>Apache Crail 1.0 incubator <a href="http://crail.incubator.apache.org/download">release</a></p>
+ <p>Apache Crail 1.0 incubator <a href="//crail.incubator.apache.org/download">release</a></p>
</li>
@@ -149,7 +149,7 @@
</ul>
-<p><a href="http://crail.incubator.apache.org/news/">Older posts…</a></p>
+<p><a href="//crail.incubator.apache.org/news/">Older posts…</a></p>
@@ -164,6 +164,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/news/index.html
----------------------------------------------------------------------
diff --git a/content/news/index.html b/content/news/index.html
index 35f79ad..ae0eede 100644
--- a/content/news/index.html
+++ b/content/news/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: News</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/news-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/news-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/news-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/news-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -79,7 +79,7 @@
<li class="shortnews">
<span class="date">August 9, 2018</span>
- <p>A new blog <a href="http://crail.incubator.apache.org/blog/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p>
+ <p>A new blog <a href="//crail.incubator.apache.org/blog/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p>
</li>
@@ -103,7 +103,7 @@
<li class="shortnews">
<span class="date">June 4, 2018</span>
- <p>Apache Crail 1.0 incubator <a href="http://crail.incubator.apache.org/download">release</a></p>
+ <p>Apache Crail 1.0 incubator <a href="//crail.incubator.apache.org/download">release</a></p>
</li>
@@ -119,7 +119,7 @@
<li class="shortnews">
<span class="date">November 23, 2017</span>
- <p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crail’s metadata performance and scalability</p>
+ <p>New blog <a href="//crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crail’s metadata performance and scalability</p>
</li>
@@ -135,7 +135,7 @@
<li class="shortnews">
<span class="date">November 17, 2017</span>
- <p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p>
+ <p>New blog <a href="//crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p>
</li>
@@ -199,7 +199,7 @@
<li class="shortnews">
<span class="date">January 17, 2017</span>
- <p>New blog <a href="http://crail.incubator.apache.org/blog/2017/01/sorting.html">post</a> about sorting on a 100Gbit/s cluster using Spark/Crail</p>
+ <p>New blog <a href="//crail.incubator.apache.org/blog/2017/01/sorting.html">post</a> about sorting on a 100Gbit/s cluster using Spark/Crail</p>
</li>
@@ -234,6 +234,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/overview/index.html
----------------------------------------------------------------------
diff --git a/content/overview/index.html b/content/overview/index.html
index d52f5ea..8e10d26 100644
--- a/content/overview/index.html
+++ b/content/overview/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: I/O Challenges</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/overview-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/overview-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/overview-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/overview-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li class="active">
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -92,7 +92,7 @@ Modern storage and network technologies such as 100Gb/s Ethernet, RDMA, NVMe fla
</li>
</ul>
-<p>In the <a href="http://crail.incubator.apache.org/blog">Blog</a> section we discuss each of those challenges in more detail.</p>
+<p>In the <a href="//crail.incubator.apache.org/blog">Blog</a> section we discuss each of those challenges in more detail.</p>
<h2 id="crail-architecture">Crail Architecture</h2>
@@ -133,7 +133,7 @@ backed with separate files inside the directory. Keyvalue nodes are similar to d
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/overview/filesystem2.png" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/overview/filesystem2.png" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -249,6 +249,6 @@ spark.crail.shuffle.sorter
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
[3/3] incubator-crail-website git commit: Publishing from
ff498052418181475486b63a7d66169591496498
Posted by pe...@apache.org.
Publishing from ff498052418181475486b63a7d66169591496498
Project: http://git-wip-us.apache.org/repos/asf/incubator-crail-website/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-crail-website/commit/5b208cb9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-crail-website/tree/5b208cb9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-crail-website/diff/5b208cb9
Branch: refs/heads/asf-site
Commit: 5b208cb94cde1c5f1e2ab45252e35aa567ebb165
Parents: 59f881c
Author: Jonas Pfefferle <pe...@apache.org>
Authored: Tue Aug 14 15:50:28 2018 +0200
Committer: Jonas Pfefferle <pe...@apache.org>
Committed: Tue Aug 14 15:50:28 2018 +0200
----------------------------------------------------------------------
content/404.html | 32 ++--
content/Gemfile.lock | 192 +++++++++++++++++--
content/blog/2017/01/firstrelease.html | 30 +--
content/blog/2017/01/sorting-post.html | 32 ++--
content/blog/2017/01/sorting.html | 50 ++---
content/blog/2017/01/spark-io.html | 30 +--
content/blog/2017/03/dataworkssummit.html | 30 +--
content/blog/2017/03/nvmf.html | 30 +--
content/blog/2017/03/nvmw.html | 30 +--
content/blog/2017/03/ofa.html | 30 +--
content/blog/2017/03/sparksummit.html | 30 +--
content/blog/2017/06/disni.html | 30 +--
content/blog/2017/08/crail-memory.html | 44 ++---
content/blog/2017/08/crail-nvme-fabrics-v1.html | 48 ++---
content/blog/2017/08/openpower.html | 30 +--
content/blog/2017/11/blog.html | 32 ++--
content/blog/2017/11/crail-metadata.html | 52 ++---
content/blog/2017/11/floss.html | 30 +--
content/blog/2017/11/iops.html | 32 ++--
content/blog/2017/11/rdmashuffle.html | 34 ++--
content/blog/2018/01/apache.html | 30 +--
content/blog/2018/06/apache-release.html | 32 ++--
content/blog/2018/06/dataworks.html | 30 +--
content/blog/2018/06/sparksummit.html | 30 +--
content/blog/2018/08/sql-p1-news.html | 32 ++--
content/blog/2018/08/sql-p1.html | 40 ++--
content/blog/index.html | 30 +--
content/blog/page2/index.html | 30 +--
content/blog/page3/index.html | 30 +--
content/blog/page4/index.html | 30 +--
content/blog/page5/index.html | 30 +--
content/community/index.html | 30 +--
content/documentation/index.html | 30 +--
content/download/index.html | 30 +--
content/feed.xml | 38 ++--
content/index.html | 38 ++--
content/news/index.html | 40 ++--
content/overview/index.html | 34 ++--
38 files changed, 799 insertions(+), 633 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/404.html
----------------------------------------------------------------------
diff --git a/content/404.html b/content/404.html
index c85c1dd..9b93dbb 100644
--- a/content/404.html
+++ b/content/404.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: 404 Not Found</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/404-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/404-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/404-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/404-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -76,7 +76,7 @@
<p>Sorry! The server can’t find that page.</p>
-<p>Please consider trying to find what you need from <a href="http://crail.incubator.apache.org/">the home page</a>.</p>
+<p>Please consider trying to find what you need from <a href="//crail.incubator.apache.org/">the home page</a>.</p>
<p>If you think something might be broken, please notify the developers at the <a href="mailto:dev@crail.incubator.apache.org">dev mailing list</a>.</p>
@@ -93,6 +93,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/Gemfile.lock
----------------------------------------------------------------------
diff --git a/content/Gemfile.lock b/content/Gemfile.lock
index f3da16c..9962c76 100644
--- a/content/Gemfile.lock
+++ b/content/Gemfile.lock
@@ -1,53 +1,207 @@
GEM
remote: https://rubygems.org/
specs:
+ activesupport (4.2.9)
+ i18n (~> 0.7)
+ minitest (~> 5.1)
+ thread_safe (~> 0.3, >= 0.3.4)
+ tzinfo (~> 1.1)
addressable (2.5.2)
public_suffix (>= 2.0.2, < 4.0)
+ coffee-script (2.4.1)
+ coffee-script-source
+ execjs
+ coffee-script-source (1.11.1)
colorator (1.1.0)
+ commonmarker (0.17.7.1)
+ ruby-enum (~> 0.5)
concurrent-ruby (1.0.5)
- em-websocket (0.5.1)
- eventmachine (>= 0.12.9)
- http_parser.rb (~> 0.6.0)
- eventmachine (1.2.5)
+ ethon (0.11.0)
+ ffi (>= 1.3.0)
+ execjs (2.7.0)
faraday (0.14.0)
multipart-post (>= 1.2, < 3)
- ffi (1.9.18)
+ ffi (1.9.21)
forwardable-extended (2.6.0)
- http_parser.rb (0.6.0)
- i18n (0.9.1)
+ gemoji (3.0.0)
+ github-pages (177)
+ activesupport (= 4.2.9)
+ github-pages-health-check (= 1.3.5)
+ jekyll (= 3.6.2)
+ jekyll-avatar (= 0.5.0)
+ jekyll-coffeescript (= 1.0.2)
+ jekyll-commonmark-ghpages (= 0.1.5)
+ jekyll-default-layout (= 0.1.4)
+ jekyll-feed (= 0.9.2)
+ jekyll-gist (= 1.4.1)
+ jekyll-github-metadata (= 2.9.3)
+ jekyll-mentions (= 1.2.0)
+ jekyll-optional-front-matter (= 0.3.0)
+ jekyll-paginate (= 1.1.0)
+ jekyll-readme-index (= 0.2.0)
+ jekyll-redirect-from (= 0.12.1)
+ jekyll-relative-links (= 0.5.2)
+ jekyll-remote-theme (= 0.2.3)
+ jekyll-sass-converter (= 1.5.0)
+ jekyll-seo-tag (= 2.3.0)
+ jekyll-sitemap (= 1.1.1)
+ jekyll-swiss (= 0.4.0)
+ jekyll-theme-architect (= 0.1.0)
+ jekyll-theme-cayman (= 0.1.0)
+ jekyll-theme-dinky (= 0.1.0)
+ jekyll-theme-hacker (= 0.1.0)
+ jekyll-theme-leap-day (= 0.1.0)
+ jekyll-theme-merlot (= 0.1.0)
+ jekyll-theme-midnight (= 0.1.0)
+ jekyll-theme-minimal (= 0.1.0)
+ jekyll-theme-modernist (= 0.1.0)
+ jekyll-theme-primer (= 0.5.2)
+ jekyll-theme-slate (= 0.1.0)
+ jekyll-theme-tactile (= 0.1.0)
+ jekyll-theme-time-machine (= 0.1.0)
+ jekyll-titles-from-headings (= 0.5.0)
+ jemoji (= 0.8.1)
+ kramdown (= 1.16.2)
+ liquid (= 4.0.0)
+ listen (= 3.0.6)
+ mercenary (~> 0.3)
+ minima (= 2.1.1)
+ nokogiri (>= 1.8.1, < 2.0)
+ rouge (= 2.2.1)
+ terminal-table (~> 1.4)
+ github-pages-health-check (1.3.5)
+ addressable (~> 2.3)
+ net-dns (~> 0.8)
+ octokit (~> 4.0)
+ public_suffix (~> 2.0)
+ typhoeus (~> 0.7)
+ html-pipeline (2.7.1)
+ activesupport (>= 2)
+ nokogiri (>= 1.4)
+ i18n (0.9.5)
concurrent-ruby (~> 1.0)
- jekyll (3.7.0)
+ jekyll (3.6.2)
addressable (~> 2.4)
colorator (~> 1.0)
- em-websocket (~> 0.5)
- i18n (~> 0.7)
jekyll-sass-converter (~> 1.0)
- jekyll-watch (~> 2.0)
+ jekyll-watch (~> 1.1)
kramdown (~> 1.14)
liquid (~> 4.0)
mercenary (~> 0.3.3)
pathutil (~> 0.9)
- rouge (>= 1.7, < 4)
+ rouge (>= 1.7, < 3)
safe_yaml (~> 1.0)
+ jekyll-avatar (0.5.0)
+ jekyll (~> 3.0)
+ jekyll-coffeescript (1.0.2)
+ coffee-script (~> 2.2)
+ coffee-script-source (~> 1.11.1)
+ jekyll-commonmark (1.1.0)
+ commonmarker (~> 0.14)
+ jekyll (>= 3.0, < 4.0)
+ jekyll-commonmark-ghpages (0.1.5)
+ commonmarker (~> 0.17.6)
+ jekyll-commonmark (~> 1)
+ rouge (~> 2)
+ jekyll-default-layout (0.1.4)
+ jekyll (~> 3.0)
jekyll-feed (0.9.2)
jekyll (~> 3.3)
jekyll-gist (1.4.1)
octokit (~> 4.2)
+ jekyll-github-metadata (2.9.3)
+ jekyll (~> 3.1)
+ octokit (~> 4.0, != 4.4.0)
+ jekyll-mentions (1.2.0)
+ activesupport (~> 4.0)
+ html-pipeline (~> 2.3)
+ jekyll (~> 3.0)
jekyll-oembed (0.0.1)
jekyll
ruby-oembed (= 0.8.8)
+ jekyll-optional-front-matter (0.3.0)
+ jekyll (~> 3.0)
jekyll-paginate (1.1.0)
+ jekyll-readme-index (0.2.0)
+ jekyll (~> 3.0)
+ jekyll-redirect-from (0.12.1)
+ jekyll (~> 3.3)
+ jekyll-relative-links (0.5.2)
+ jekyll (~> 3.3)
+ jekyll-remote-theme (0.2.3)
+ jekyll (~> 3.5)
+ rubyzip (>= 1.2.1, < 3.0)
+ typhoeus (>= 0.7, < 2.0)
jekyll-sass-converter (1.5.0)
sass (~> 3.4)
- jekyll-watch (2.0.0)
+ jekyll-seo-tag (2.3.0)
+ jekyll (~> 3.3)
+ jekyll-sitemap (1.1.1)
+ jekyll (~> 3.3)
+ jekyll-swiss (0.4.0)
+ jekyll-theme-architect (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-cayman (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-dinky (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-hacker (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-leap-day (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-merlot (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-midnight (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-minimal (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-modernist (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-primer (0.5.2)
+ jekyll (~> 3.5)
+ jekyll-github-metadata (~> 2.9)
+ jekyll-seo-tag (~> 2.2)
+ jekyll-theme-slate (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-tactile (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-theme-time-machine (0.1.0)
+ jekyll (~> 3.5)
+ jekyll-seo-tag (~> 2.0)
+ jekyll-titles-from-headings (0.5.0)
+ jekyll (~> 3.3)
+ jekyll-watch (1.5.1)
listen (~> 3.0)
+ jemoji (0.8.1)
+ activesupport (~> 4.0, >= 4.2.9)
+ gemoji (~> 3.0)
+ html-pipeline (~> 2.2)
+ jekyll (>= 3.0)
kramdown (1.16.2)
liquid (4.0.0)
listen (3.0.6)
rb-fsevent (>= 0.9.3)
rb-inotify (>= 0.9.7)
mercenary (0.3.6)
+ mini_portile2 (2.3.0)
+ minima (2.1.1)
+ jekyll (~> 3.3)
+ minitest (5.11.3)
multipart-post (2.0.0)
+ net-dns (0.8.0)
+ nokogiri (1.8.2)
+ mini_portile2 (~> 2.3.0)
octokit (4.8.0)
sawyer (~> 0.8.0, >= 0.5.3)
pathutil (0.16.1)
@@ -57,7 +211,10 @@ GEM
rb-inotify (0.9.10)
ffi (>= 0.5.0, < 2)
rouge (2.2.1)
+ ruby-enum (0.7.1)
+ i18n
ruby-oembed (0.8.8)
+ rubyzip (1.2.1)
safe_yaml (1.0.4)
sass (3.5.5)
sass-listen (~> 4.0.0)
@@ -67,11 +224,20 @@ GEM
sawyer (0.8.1)
addressable (>= 2.3.5, < 2.6)
faraday (~> 0.8, < 1.0)
+ terminal-table (1.8.0)
+ unicode-display_width (~> 1.1, >= 1.1.1)
+ thread_safe (0.3.6)
+ typhoeus (0.8.0)
+ ethon (>= 0.8.0)
+ tzinfo (1.2.5)
+ thread_safe (~> 0.1)
+ unicode-display_width (1.3.0)
PLATFORMS
ruby
DEPENDENCIES
+ github-pages
jekyll-feed
jekyll-gist
jekyll-oembed
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/01/firstrelease.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/01/firstrelease.html b/content/blog/2017/01/firstrelease.html
index ee06806..cc52338 100644
--- a/content/blog/2017/01/firstrelease.html
+++ b/content/blog/2017/01/firstrelease.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Firstrelease</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/firstrelease-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/firstrelease-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/firstrelease-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/firstrelease-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/01/sorting-post.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/01/sorting-post.html b/content/blog/2017/01/sorting-post.html
index 26949df..9ba4634 100644
--- a/content/blog/2017/01/sorting-post.html
+++ b/content/blog/2017/01/sorting-post.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Sorting Post</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/sorting-post-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/sorting-post-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/sorting-post-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/sorting-post-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -74,7 +74,7 @@
<h2>Sorting Post</h2>
- <p>New blog <a href="http://crail.incubator.apache.org/blog/2017/01/sorting.html">post</a> about sorting on a 100Gbit/s cluster using Spark/Crail</p>
+ <p>New blog <a href="//crail.incubator.apache.org/blog/2017/01/sorting.html">post</a> about sorting on a 100Gbit/s cluster using Spark/Crail</p>
<br>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/01/sorting.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/01/sorting.html b/content/blog/2017/01/sorting.html
index 1d84fbd..dca63b1 100644
--- a/content/blog/2017/01/sorting.html
+++ b/content/blog/2017/01/sorting.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Sorting on a 100Gbit/s Cluster using Spark/Crail</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/sorting-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/sorting-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/sorting-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/sorting-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -190,7 +190,7 @@ A Spark sorting job consists of two phases. The first phase is a mapping or cla
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sort/terasort_pipeline.png" width="490" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sort/terasort_pipeline.png" width="490" /></div>
<p><br /><br /></p>
<h3 id="using-vanilla-spark">Using Vanilla Spark</h3>
@@ -202,7 +202,7 @@ The first question we are interested in is to what extent such a sorting benchma
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sort/net_vanilla.svg" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sort/net_vanilla.svg" /></div>
<p><br /><br /></p>
<div style="text-align: justify">
@@ -212,7 +212,7 @@ The poor network usage matches with the general observation we made in our previ
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sort/cpu_network.svg" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sort/cpu_network.svg" /></div>
<p><br /><br /></p>
<div style="text-align: justify">
@@ -225,12 +225,12 @@ Consequently, to improve the runtime of the sorting benchmark and to make good u
<div style="text-align: justify">
<p>
-An overview of the Crail shuffler is provided in the <a href="http://crail.incubator.apache.org/docs">documentation section</a>. The main difference between the Crail shuffler and the Spark built-in shuffler lies in the way data from the network is processed in a reduce task. The Spark shuffler is based on TCP sockets, thus, many CPU instructions are necessary to bring the data from the networking interface to the buffer inside Spark. In contrast, the Crail shuffler shares shuffle data through the Crail file system, and therefore data is transferred directly via DMA from the network interface to the Spark shuffle buffer within the JVM.
+An overview of the Crail shuffler is provided in the <a href="//crail.incubator.apache.org/docs">documentation section</a>. The main difference between the Crail shuffler and the Spark built-in shuffler lies in the way data from the network is processed in a reduce task. The Spark shuffler is based on TCP sockets, thus, many CPU instructions are necessary to bring the data from the networking interface to the buffer inside Spark. In contrast, the Crail shuffler shares shuffle data through the Crail file system, and therefore data is transferred directly via DMA from the network interface to the Spark shuffle buffer within the JVM.
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sort/shuffle_rdma.png" width="470" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sort/shuffle_rdma.png" width="470" /></div>
<p><br /><br /></p>
<div style="text-align: justify">
@@ -239,7 +239,7 @@ During the map phase, the Crail shuffler organizes each key range in a set of Cr
</p>
<p>
-As illustrated in the <a href="http://crail.incubator.apache.org/docs">documentation section</a>, the Crail shuffler allows applications to use their own custom serializer and sorter. The recommended serializer for Spark is Kryo, which is a generic serializer. Being generic, however, comes at a cost. Specifically, Kryo requires more type information to be stored along with the serialized data than a custom serializer would need, and also the parsing is more complex for a generic serializer. On top of this, Kryo also comes with its own buffering, introducing additional memory copies. In our benchmark, we use a custom serializer that takes advantage of the fact that the data consists of fixed size key/value pairs. The custom serializer further avoids extra buffering and directly interfaces with Crail file system streams when reading and writing data.
+As illustrated in the <a href="//crail.incubator.apache.org/docs">documentation section</a>, the Crail shuffler allows applications to use their own custom serializer and sorter. The recommended serializer for Spark is Kryo, which is a generic serializer. Being generic, however, comes at a cost. Specifically, Kryo requires more type information to be stored along with the serialized data than a custom serializer would need, and also the parsing is more complex for a generic serializer. On top of this, Kryo also comes with its own buffering, introducing additional memory copies. In our benchmark, we use a custom serializer that takes advantage of the fact that the data consists of fixed size key/value pairs. The custom serializer further avoids extra buffering and directly interfaces with Crail file system streams when reading and writing data.
</p>
<p>
As with serialization, the Spark built-in sorter is a general purpose TimSort that can sort arbitrary collections of comparable objects. In our benchmark, we instruct the Crail shuffler to use a Radix sorter instead. The Radix sorter cannot be applied to arbitrary objects but works well for keys of a fixed byte length. The standard pipeline of a reduce task is to first deserialize the data and then sort it. In the particular configuration of the Crail shuffler, we turn these two steps around and first sort the data and deserialize it later. This is possible because the data is read into a contiguous off-heap buffer that can be sorted almost in-place.
@@ -255,13 +255,13 @@ The figure below shows the overall performance of Spark/Crail vs Spark/Vanilla o
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sort/performance_overall.png" width="470" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sort/performance_overall.png" width="470" /></div>
<p><br /><br /></p>
<p>One key question of interest is about the network usage of the Crail shuffler during the sorting benchmark. In the figure below, we show the data rate at which the different reduce tasks fetch data from the network. Each point in the figure corresponds to one reduce task. In our configuration, we run 3 Spark executors per node and 5 Spark cores per executor. Thus, 1920 reduce tasks are running concurrently (out of 6400 reduce tasks in total) generating a cluster-wide all-to-all traffic of about 70Gbit/s per node during that phase.</p>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sort/multiread.svg" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sort/multiread.svg" /></div>
<p><br /><br /></p>
<p>In this blog post, we have shown that Crail successfully manages to translate the raw network performance into actual workload level gains. The exercise with TeraSort as an application validates the design decisions we made in Crail. Stay tuned for more results with different workloads and hardware configurations.</p>
@@ -271,8 +271,8 @@ The figure below shows the overall performance of Spark/Crail vs Spark/Vanilla o
<p>All the components required to run the sorting benchmark using Spark/Crail are open source. Here is some guidance how to run the benchmark:</p>
<ul>
- <li>Build and deploy Crail using the instructions at <a href="http://crail.incubator.apache.org/documentation#crail">documentation</a></li>
- <li>Enable the Crail shuffler for Spark by building Spark-IO using the instructions at <a href="http://crail.incubator.apache.org/documentation#spark">documentation</a></li>
+ <li>Build and deploy Crail using the instructions at <a href="//crail.incubator.apache.org/documentation#crail">documentation</a></li>
+ <li>Enable the Crail shuffler for Spark by building Spark-IO using the instructions at <a href="//crail.incubator.apache.org/documentation#spark">documentation</a></li>
<li>Configure the DRAM storage tier of Crail so that all the shuffle data fits into the DRAM tier.</li>
<li>Build the sorting benchmark using the instructions on <a href="https://github.com/zrlio/crail-terasort">GitHub</a></li>
<li>Make sure you have the custom serializer and sorter specified in spark-defaults.conf</li>
@@ -329,6 +329,6 @@ s.setAttribute('data-timestamp', +new Date());
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/01/spark-io.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/01/spark-io.html b/content/blog/2017/01/spark-io.html
index 992da63..fc9f132 100644
--- a/content/blog/2017/01/spark-io.html
+++ b/content/blog/2017/01/spark-io.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Spark Io</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/spark-io-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/spark-io-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/spark-io-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/spark-io-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/03/dataworkssummit.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/03/dataworkssummit.html b/content/blog/2017/03/dataworkssummit.html
index 9e9956e..5fc0fee 100644
--- a/content/blog/2017/03/dataworkssummit.html
+++ b/content/blog/2017/03/dataworkssummit.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Dataworkssummit</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/dataworkssummit-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/dataworkssummit-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/dataworkssummit-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/dataworkssummit-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/03/nvmf.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/03/nvmf.html b/content/blog/2017/03/nvmf.html
index 567204b..7b05314 100644
--- a/content/blog/2017/03/nvmf.html
+++ b/content/blog/2017/03/nvmf.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Nvmf</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/nvmf-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/nvmf-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/nvmf-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/nvmf-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/03/nvmw.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/03/nvmw.html b/content/blog/2017/03/nvmw.html
index af32d5c..8c9211e 100644
--- a/content/blog/2017/03/nvmw.html
+++ b/content/blog/2017/03/nvmw.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Nvmw</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/nvmw-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/nvmw-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/nvmw-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/nvmw-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/03/ofa.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/03/ofa.html b/content/blog/2017/03/ofa.html
index 7a48aae..498a05b 100644
--- a/content/blog/2017/03/ofa.html
+++ b/content/blog/2017/03/ofa.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Ofa</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/ofa-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/ofa-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/ofa-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/ofa-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/03/sparksummit.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/03/sparksummit.html b/content/blog/2017/03/sparksummit.html
index 971bb1a..d8f203e 100644
--- a/content/blog/2017/03/sparksummit.html
+++ b/content/blog/2017/03/sparksummit.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Sparksummit</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/06/disni.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/06/disni.html b/content/blog/2017/06/disni.html
index 09bfb61..4e2fbdb 100644
--- a/content/blog/2017/06/disni.html
+++ b/content/blog/2017/06/disni.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Disni</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/disni-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/disni-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/disni-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/disni-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/08/crail-memory.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/08/crail-memory.html b/content/blog/2017/08/crail-memory.html
index e8671b6..7effb61 100644
--- a/content/blog/2017/08/crail-memory.html
+++ b/content/blog/2017/08/crail-memory.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Crail Storage Performance -- Part I: DRAM</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/crail-memory-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/crail-memory-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/crail-memory-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/crail-memory-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -144,7 +144,7 @@ One challenge with file read/write operations is to avoid blocking in case block
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/anatomy.png" width="420" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/anatomy.png" width="420" /></div>
<p><br /></p>
<div style="text-align: justify">
<p>
@@ -180,8 +180,8 @@ The figure below illustrates the sequential write (top) and read (bottom) perfor
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/write.svg" width="550" /></div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/read.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/write.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/read.svg" width="550" /></div>
<p><br /><br /></p>
<div style="text-align: justify">
<p>
@@ -192,8 +192,8 @@ Note that both figures show single-client performance numbers. With Crail being
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/crail-groupby.svg" width="550" /></div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/spark-groupby.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/crail-groupby.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/spark-groupby.svg" width="550" /></div>
<h3 id="random-read-latency">Random Read Latency</h3>
@@ -209,7 +209,7 @@ Typically, distributed storage systems are either built for sequential access to
The figure below illustrates the latencies of get() operations for different key/value sizes and compares them to the latencies we obtained with RAMCloud for the same type of operations (measured using RAMClouds C and Java APIs). RAMCloud is a low-latency key/value store implemented using RDMA. RAMCloud actually provides durable storage by asynchronously replicating data onto backup devices. However, at any point in time all the data is held in DRAM and read requests will be served from DRAM directly. Up to our knowledge, RAMCloud is the fastest key/value store that is (a) available open source and (b) can be deployed in practice as a storage platform for applications. Other similar RDMA-based storage systems we looked at, like FaRM or HERD, are either not open source or they do not provide a clean separation between storage system, API and clients.
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/latency.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/latency.svg" width="550" /></div>
<div style="text-align: justify">
<p>
@@ -223,7 +223,7 @@ The latency advantages of Crail are beneficial also at the application level. Th
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-memory/cdf-broadcast-128-read.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-memory/cdf-broadcast-128-read.svg" width="550" /></div>
<div style="text-align: justify">
<p>
@@ -271,6 +271,6 @@ s.setAttribute('data-timestamp', +new Date());
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/08/crail-nvme-fabrics-v1.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/08/crail-nvme-fabrics-v1.html b/content/blog/2017/08/crail-nvme-fabrics-v1.html
index 9ecc234..1a77d9d 100644
--- a/content/blog/2017/08/crail-nvme-fabrics-v1.html
+++ b/content/blog/2017/08/crail-nvme-fabrics-v1.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Crail Storage Performance -- Part II: NVMf</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/crail-nvme-fabrics-v1-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/crail-nvme-fabrics-v1-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/crail-nvme-fabrics-v1-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/crail-nvme-fabrics-v1-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -160,7 +160,7 @@ The main take away from this plot is that the time it takes to perform a random
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/latency.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-nvmf/latency.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -175,16 +175,16 @@ The second plot shows sequential read and write throughput with a transfer size
</code></pre></div></div>
<div style="text-align: justify">
<p>
-For sequential operations in Crail, metadata fetching is inlined with data operations as described in the <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a> blog. This is possible as long as the data transfer has a lower latency than the metadata RPC, which is typically the case. As a consequence, our NVMf storage tier reaches the same throughput as the native SPDK benchmark (device limit).
+For sequential operations in Crail, metadata fetching is inlined with data operations as described in the <a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a> blog. This is possible as long as the data transfer has a lower latency than the metadata RPC, which is typically the case. As a consequence, our NVMf storage tier reaches the same throughput as the native SPDK benchmark (device limit).
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-nvmf/throughput.svg" width="550" /></div>
<h3 id="sequential-throughput">Sequential Throughput</h3>
<div style="text-align: justify">
<p>
-Let us look at the sequential read and write throughput for buffered and direct streams and compare them to a buffered Crail stream on DRAM. All benchmarks are single thread/client performed against 8 storage nodes with 4 drives each, cf. configuration above. In this benchmark we use 32 outstanding operations for the NVMf storage tier buffered stream experiments by using a buffer size of 16MB and a slice size of 512KB, cf. <a href="http://crail.incubator.apache.org/blog/2017/07/crail-memory.html">part I</a>. The buffered stream reaches line speed at a transfer size of around 1KB and shows only slightly slower performance when compared to the DRAM tier buffered stream. However we are only using 2 outstanding operations with the DRAM tier to achieve these results. Basically for sizes smaller than 1KB the buffered stream is limited by the copy speed to fill the application buffer. The direct stream reaches line speed at around 128KB with 128 outstanding operations. Here no copy operati
on is performed for transfer size greater than 512Byte (sector size). The command to run the Crail buffered stream benchmark:
+Let us look at the sequential read and write throughput for buffered and direct streams and compare them to a buffered Crail stream on DRAM. All benchmarks are single thread/client performed against 8 storage nodes with 4 drives each, cf. configuration above. In this benchmark we use 32 outstanding operations for the NVMf storage tier buffered stream experiments by using a buffer size of 16MB and a slice size of 512KB, cf. <a href="//crail.incubator.apache.org/blog/2017/07/crail-memory.html">part I</a>. The buffered stream reaches line speed at a transfer size of around 1KB and shows only slightly slower performance when compared to the DRAM tier buffered stream. However we are only using 2 outstanding operations with the DRAM tier to achieve these results. Basically for sizes smaller than 1KB the buffered stream is limited by the copy speed to fill the application buffer. The direct stream reaches line speed at around 128KB with 128 outstanding operations. Here no copy operation is
performed for transfer size greater than 512Byte (sector size). The command to run the Crail buffered stream benchmark:
</p>
</div>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t read -s <size> -k <iterations> -w 32 -f /tmp.dat
@@ -193,7 +193,7 @@ Let us look at the sequential read and write throughput for buffered and direct
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t readAsync -s <size> -k <iterations> -b 128 -w 32 -f /tmp.dat
</code></pre></div></div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/throughput2.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-nvmf/throughput2.svg" width="550" /></div>
<h3 id="random-read-latency">Random Read Latency</h3>
@@ -203,16 +203,16 @@ Random read latency is limited by the flash technology and we currently see arou
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/latency2.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-nvmf/latency2.svg" width="550" /></div>
<h3 id="tiering-dram---nvmf">Tiering DRAM - NVMf</h3>
<div style="text-align: justify">
<p>
-In this paragraph we show how Crail can leverage flash memory when there is not sufficient DRAM available in the cluster to hold all the data. As described in the <a href="http://crail.incubator.apache.org/overview/">overview</a> section, if you have multiple storage tiers deployed in Crail, e.g. the DRAM tier and the NVMf tier, Crail by default first uses up all available resources of the faster tier. Basically a remote resource of a faster tier (e.g. remote DRAM) is preferred over a slower local resource (e.g., local flash), motivated by the fast network. This is what we call horizontal tiering.
+In this paragraph we show how Crail can leverage flash memory when there is not sufficient DRAM available in the cluster to hold all the data. As described in the <a href="//crail.incubator.apache.org/overview/">overview</a> section, if you have multiple storage tiers deployed in Crail, e.g. the DRAM tier and the NVMf tier, Crail by default first uses up all available resources of the faster tier. Basically a remote resource of a faster tier (e.g. remote DRAM) is preferred over a slower local resource (e.g., local flash), motivated by the fast network. This is what we call horizontal tiering.
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/crail_tiering.png" width="500" vspace="10" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-nvmf/crail_tiering.png" width="500" vspace="10" /></div>
<p><br /></p>
<div style="text-align: justify">
<p>
@@ -220,7 +220,7 @@ In the following 200G Terasort experiment we gradually limit the DRAM resources
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-nvmf/tiering.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-nvmf/tiering.svg" width="550" /></div>
<p>To summarize, in this blog we have shown that the NVMf storage backend for Crail – due to its efficient user-level implementation – offers latencies and throughput very close to the hardware speed. The Crail NVMf storage tier can be used conveniently in combination with the Crail DRAM tier to either save cost or to handle situations where the available DRAM is not sufficient to store the working set of a data processing workload.</p>
@@ -264,6 +264,6 @@ s.setAttribute('data-timestamp', +new Date());
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/08/openpower.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/08/openpower.html b/content/blog/2017/08/openpower.html
index 001e815..a5897d2 100644
--- a/content/blog/2017/08/openpower.html
+++ b/content/blog/2017/08/openpower.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Openpower</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/openpower-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/openpower-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/openpower-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/openpower-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
[2/3] incubator-crail-website git commit: Publishing from
ff498052418181475486b63a7d66169591496498
Posted by pe...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/11/blog.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/11/blog.html b/content/blog/2017/11/blog.html
index aaaa15e..66a31e4 100644
--- a/content/blog/2017/11/blog.html
+++ b/content/blog/2017/11/blog.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Blog</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -74,7 +74,7 @@
<h2>Blog</h2>
- <p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p>
+ <p>New blog <a href="//crail.incubator.apache.org/blog/2017/11/rdmashuffle.html">post</a> about SparkRDMA and Crail shuffle plugins</p>
<br>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/11/crail-metadata.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/11/crail-metadata.html b/content/blog/2017/11/crail-metadata.html
index 2aa4644..cdd48f4 100644
--- a/content/blog/2017/11/crail-metadata.html
+++ b/content/blog/2017/11/crail-metadata.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Crail Storage Performance -- Part III: Metadata</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/crail-metadata-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/crail-metadata-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/crail-metadata-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/crail-metadata-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -112,11 +112,11 @@ This is part III of our series of posts discussing Crail's raw storage performan
<div style="text-align: justify">
<p>
-As described in <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">part I</a>, Crail data operations are composed of actual data transfers and metadata operations. Examples of metadata operations are operations for creating or modifying the state of a file, or operations to lookup the storage server that stores a particular range (block) of a file. In Crail, all the metadata is managed by the namenode(s) (as opposed to the data which is managed by the storage nodes). Clients interact with Crail namenodes via Remote Procedure Calls (RPCs). Crail supports multiple RPC protocols for different types of networks and also offers a pluggable RPC interface so that new RPC bindings can be implemented easily. On RDMA networks, the default DaRPC (<a href="https://dl.acm.org/citation.cfm?id=2670994">DaRPC paper</a>, <a href="http://github.com/zrlio/darpc">DaRPC GitHub</a>) based RPC binding provides the best performance. The figure below gives an overview of the Crail me
tadata processing in a DaRPC configuration.
+As described in <a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">part I</a>, Crail data operations are composed of actual data transfers and metadata operations. Examples of metadata operations are operations for creating or modifying the state of a file, or operations to lookup the storage server that stores a particular range (block) of a file. In Crail, all the metadata is managed by the namenode(s) (as opposed to the data which is managed by the storage nodes). Clients interact with Crail namenodes via Remote Procedure Calls (RPCs). Crail supports multiple RPC protocols for different types of networks and also offers a pluggable RPC interface so that new RPC bindings can be implemented easily. On RDMA networks, the default DaRPC (<a href="https://dl.acm.org/citation.cfm?id=2670994">DaRPC paper</a>, <a href="http://github.com/zrlio/darpc">DaRPC GitHub</a>) based RPC binding provides the best performance. The figure below gives an overview of the Crail metadat
a processing in a DaRPC configuration.
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/rpc.png" width="480" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/rpc.png" width="480" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -129,10 +129,10 @@ Crail supports partitioning of metadata across several namenods. Thereby, metada
<div style="text-align: justify">
<p>
-In two of the previous blogs (<a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>, <a href="http://crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">NVMf</a>) we have already shown that Crail metadata operations are very low latency. Essentially a single metadata operation issued by a remote client takes 5-6 microseconds, which is only slightly more than the raw network latency of the RDMA network fabric. In this blog, we want to explore the scalability of Crail's metadata management, that is, the number of clients Crail can support, or how Crail scales as the cluster size increases. The level of scalability of Crail is mainly determined by the number of metadata operations Crail can process concurrently, a metric that is often referred to as IOPS. The higher the number of IOPS the system can handle, the more clients can concurrently use Crail without performance loss.
+In two of the previous blogs (<a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>, <a href="//crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">NVMf</a>) we have already shown that Crail metadata operations are very low latency. Essentially a single metadata operation issued by a remote client takes 5-6 microseconds, which is only slightly more than the raw network latency of the RDMA network fabric. In this blog, we want to explore the scalability of Crail's metadata management, that is, the number of clients Crail can support, or how Crail scales as the cluster size increases. The level of scalability of Crail is mainly determined by the number of metadata operations Crail can process concurrently, a metric that is often referred to as IOPS. The higher the number of IOPS the system can handle, the more clients can concurrently use Crail without performance loss.
</p>
<p>
-An important metadata operation is ''getFile()'', which is used by clients to lookup the status of a file (whether the file exists, what size it has, etc.). The ''getFile()'' operation is served by Crail's fast lock-free map and in spirit is very similar to the ''getBlock()'' metadata operation (used by clients to query which storage nodes holds a particular block). In a typical Crail use case, ''getFile()'' and ''getBlock()'' are responsible for the peak metadata load at a namenode. In this experiment, we measure the achievable IOPS on the server side in an artificial configuration with many clients distributed across the cluster issuing ''getFile()'' in a tight loop. Note that the client side RPC interface in Crail is asynchronous, thus, clients can issue multiple metadata operations without blocking while asynchronously waiting for the result. In the experiments below, each client may have a maximum of 128 ''getFile()'' operations outstanding at any point in time. In a practical
scenario, Crail clients may also have multiple metadata operations in flight either because clients are shared by different cores, or because Crail interleaves metadata and data operations (see <a href="http://crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>). What makes the benchmark artificial is that clients exclusively focus on generating load for the namenode and thereby are neither performing data operations nor are they doing any compute. The basic command of the benchmark as executed by each of the individual clients is given by the following command:
+An important metadata operation is ''getFile()'', which is used by clients to lookup the status of a file (whether the file exists, what size it has, etc.). The ''getFile()'' operation is served by Crail's fast lock-free map and in spirit is very similar to the ''getBlock()'' metadata operation (used by clients to query which storage nodes holds a particular block). In a typical Crail use case, ''getFile()'' and ''getBlock()'' are responsible for the peak metadata load at a namenode. In this experiment, we measure the achievable IOPS on the server side in an artificial configuration with many clients distributed across the cluster issuing ''getFile()'' in a tight loop. Note that the client side RPC interface in Crail is asynchronous, thus, clients can issue multiple metadata operations without blocking while asynchronously waiting for the result. In the experiments below, each client may have a maximum of 128 ''getFile()'' operations outstanding at any point in time. In a practical
scenario, Crail clients may also have multiple metadata operations in flight either because clients are shared by different cores, or because Crail interleaves metadata and data operations (see <a href="//crail.incubator.apache.org/blog/2017/08/crail-memory.html">DRAM</a>). What makes the benchmark artificial is that clients exclusively focus on generating load for the namenode and thereby are neither performing data operations nor are they doing any compute. The basic command of the benchmark as executed by each of the individual clients is given by the following command:
</p>
</div>
<div class="highlighter-rouge"><div class="highlight"><pre class="highlight"><code>./bin/crail iobench -t getMultiFileAsync -f / -k 10000000 -b 128
@@ -175,7 +175,7 @@ The line of the raw number of IOPS, labeled ''ib send'' is shown in the same gra
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/namenode_ibsend_iops64.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
<p>
@@ -192,7 +192,7 @@ To increase the number of IOPS the overall system can handle, we allow starting
</p>
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/namenode_multi64.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -220,7 +220,7 @@ namenodes happening, which should lead to linear scalability.
<div style="text-align: justify">
<p>
Let us look at a concrete application, which ideally runs on a large cluster:
-TeraSort. In a previous blog, <a href="http://crail.incubator.apache.org/blog/2017/01/sorting.html">sorting</a>,
+TeraSort. In a previous blog, <a href="//crail.incubator.apache.org/blog/2017/01/sorting.html">sorting</a>,
we analyze performance characteristics of TeraSort on Crail on a big cluster
of 128 nodes, where we run 384 executors in total. This already proves that
Crail can at least handle 384 clients. Now we analyze the theoretical number
@@ -254,7 +254,7 @@ namenode over the elapsed runtime of the TeraSort application.
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/terasort_iops.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -418,7 +418,7 @@ plot shows the number of IOPS relative to the number of clients.
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/namenode_hdfs_iops.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -472,7 +472,7 @@ the blog with the latest numbers as soon as the bug is fixed.
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/ramcloud_iops.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -503,7 +503,7 @@ of 30Mio/s with 4 namenodes).
</div>
<p><br /></p>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/crail-metadata/max_iops_crail_hdfs_ramcloud.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -565,6 +565,6 @@ s.setAttribute('data-timestamp', +new Date());
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/11/floss.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/11/floss.html b/content/blog/2017/11/floss.html
index a9ea68a..7bdd6cc 100644
--- a/content/blog/2017/11/floss.html
+++ b/content/blog/2017/11/floss.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Floss</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/floss-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/floss-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/floss-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/floss-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/11/iops.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/11/iops.html b/content/blog/2017/11/iops.html
index 0b40122..5b91957 100644
--- a/content/blog/2017/11/iops.html
+++ b/content/blog/2017/11/iops.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Iops</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/iops-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/iops-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/iops-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/iops-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -74,7 +74,7 @@
<h2>Iops</h2>
- <p>New blog <a href="http://crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crail’s metadata performance and scalability</p>
+ <p>New blog <a href="//crail.incubator.apache.org/blog/2017/11/crail-metadata.html">post</a> about Crail’s metadata performance and scalability</p>
<br>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2017/11/rdmashuffle.html
----------------------------------------------------------------------
diff --git a/content/blog/2017/11/rdmashuffle.html b/content/blog/2017/11/rdmashuffle.html
index 9bed165..af4f0f0 100644
--- a/content/blog/2017/11/rdmashuffle.html
+++ b/content/blog/2017/11/rdmashuffle.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Spark Shuffle: SparkRDMA vs Crail</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/rdmashuffle-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/rdmashuffle-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/rdmashuffle-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/rdmashuffle-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -125,7 +125,7 @@ In contrast, the Crail shuffler plugin takes a more holistic approach and levera
<p>Lets start by quantitatively assessing performance gains from the Crail shuffle plugin and SparkRDMA. As described above, SparkRDMA can be operated in two different modes. Users decide which mode to use by selecting a particular type of shuffle writer (spark.shuffle.rdma.shuffleWriterMethod). The Wrapper shuffle writer writes shuffle data to files between the stages, the Chunked shuffle writer stores shuffle data in memory. We evaluate both writer methods for terasort and SQL equijoin.
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/rdma-shuffle/terasort.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/rdma-shuffle/terasort.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
<p>
@@ -135,7 +135,7 @@ First we run <a href="https://github.com/zrlio/crail-spark-terasort">terasort</a
The plot above shows runtimes of the various configuration we run with terasort. SparkRDMA with the Wrapper shuffle writer performance slightly better (3-4%) than vanilla Spark whereas the Chunked shuffle writer shows a 30% overhead. On a quick inspection we found that this overhead stems from memory allocation and registration for the shuffle data that is kept in memory between the stages. Compared to vanilla Spark, Crail's shuffle plugin shows performance improvement of around 235%.
</p>
</div>
-<div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/rdma-shuffle/sql.svg" width="550" /></div>
+<div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/rdma-shuffle/sql.svg" width="550" /></div>
<p><br /></p>
<div style="text-align: justify">
@@ -197,6 +197,6 @@ s.setAttribute('data-timestamp', +new Date());
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2018/01/apache.html
----------------------------------------------------------------------
diff --git a/content/blog/2018/01/apache.html b/content/blog/2018/01/apache.html
index ad146b7..e3abaf2 100644
--- a/content/blog/2018/01/apache.html
+++ b/content/blog/2018/01/apache.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Apache</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/apache-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/apache-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/apache-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/apache-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2018/06/apache-release.html
----------------------------------------------------------------------
diff --git a/content/blog/2018/06/apache-release.html b/content/blog/2018/06/apache-release.html
index 1f7e8ae..23f40a2 100644
--- a/content/blog/2018/06/apache-release.html
+++ b/content/blog/2018/06/apache-release.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Apache Release</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/apache-release-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/apache-release-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/apache-release-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/apache-release-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -74,7 +74,7 @@
<h2>Apache Release</h2>
- <p>Apache Crail 1.0 incubator <a href="http://crail.incubator.apache.org/download">release</a></p>
+ <p>Apache Crail 1.0 incubator <a href="//crail.incubator.apache.org/download">release</a></p>
<br>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2018/06/dataworks.html
----------------------------------------------------------------------
diff --git a/content/blog/2018/06/dataworks.html b/content/blog/2018/06/dataworks.html
index 41ec39e..c278119 100644
--- a/content/blog/2018/06/dataworks.html
+++ b/content/blog/2018/06/dataworks.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Dataworks</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/dataworks-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/dataworks-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/dataworks-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/dataworks-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2018/06/sparksummit.html
----------------------------------------------------------------------
diff --git a/content/blog/2018/06/sparksummit.html b/content/blog/2018/06/sparksummit.html
index 651e681..d6afa64 100644
--- a/content/blog/2018/06/sparksummit.html
+++ b/content/blog/2018/06/sparksummit.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Sparksummit</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/sparksummit-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2018/08/sql-p1-news.html
----------------------------------------------------------------------
diff --git a/content/blog/2018/08/sql-p1-news.html b/content/blog/2018/08/sql-p1-news.html
index 93cece5..a3b81cc 100644
--- a/content/blog/2018/08/sql-p1-news.html
+++ b/content/blog/2018/08/sql-p1-news.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Sql P1 News</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/sql-p1-news-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/sql-p1-news-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/sql-p1-news-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/sql-p1-news-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -74,7 +74,7 @@
<h2>Sql P1 News</h2>
- <p>A new blog <a href="http://crail.incubator.apache.org/blog/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p>
+ <p>A new blog <a href="//crail.incubator.apache.org/blog/2018/08/sql-p1.html">post</a> discussing file formats performance is now online</p>
<br>
@@ -88,6 +88,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/2018/08/sql-p1.html
----------------------------------------------------------------------
diff --git a/content/blog/2018/08/sql-p1.html b/content/blog/2018/08/sql-p1.html
index 885bafd..285f8f5 100644
--- a/content/blog/2018/08/sql-p1.html
+++ b/content/blog/2018/08/sql-p1.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: SQL Performance: Part 1 - Input File Formats</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/sql-p1-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/sql-p1-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/sql-p1-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/sql-p1-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -114,11 +114,11 @@ This is the first blog post in a multi-part series where we will focus on relati
<p>In a typical cloud-based relational data processing setup, the input data is stored on an external data storage solution like HDFS or AWS S3. Data tables and their associated schema are converted into a storage-friendly format for optimal performance. Examples of some popular and familiar file formats are <a href="https://parquet.apache.org/">Apache Parquet</a>, <a href="https://orc.apache.org/">Apache ORC</a>, <a href="https://avro.apache.org/">Apache Avro</a>, <a href="https://en.wikipedia.org/wiki/JSON">JSON</a>, etc. More recently, <a href="https://arrow.apache.org/">Apache Arrow</a> has been introduced to standardize the in-memory columnar data representation between multiple frameworks. There is no one size fits all as all these formats have their own strengths, weaknesses, and features. In this blog, we are specifically interested in the performance of these formats on modern high-performance networking and storage devices.</p>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/outline.svg" width="550" /><figcaption>Figure 1: The benchmarking setup with HDFS and file formats on a 100 Gbps network with NVMe flash devices. All formats contains routines for compression, encoding, and value materialization with associated I/O buffer management and data copies routines.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/outline.svg" width="550" /><figcaption>Figure 1: The benchmarking setup with HDFS and file formats on a 100 Gbps network with NVMe flash devices. All formats contains routines for compression, encoding, and value materialization with associated I/O buffer management and data copies routines.<p></p></figcaption></div></figure>
<p>To benchmark the performance of file formats, we wrote a set of micro-benchmarks which are available at <a href="https://github.com/zrlio/fileformat-benchmarks">https://github.com/zrlio/fileformat-benchmarks</a>. We cannot use typical SQL micro-benchmarks because every SQL engine has its own favorite file format, on which it performs the best. Hence, in order to ensure parity, we decoupled the performance of reading the input file format from the SQL query processing by writing simple table reading micro-benchmarks. Our benchmark reads in the store_sales table from the TPC-DS dataset (scale factor 100), and calculates a sum of values present in the table. The table contains 23 columns of integers, doubles, and longs.</p>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/performance-all.svg" width="550" /><figcaption>Figure 2: Performance of JSON, Avro, Parquet, ORC, and Arrow on NVMe devices over a 100 Gbps network.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/performance-all.svg" width="550" /><figcaption>Figure 2: Performance of JSON, Avro, Parquet, ORC, and Arrow on NVMe devices over a 100 Gbps network.<p></p></figcaption></div></figure>
<p>We evaluate the performance of the benchmark on a 3 node HDFS cluster connected using 100 Gbps RoCE. One datanode in HDFS contains 4 NVMe devices with a collective aggregate bandwidth of 12.5 GB/sec (equals to 100 Gbps, hence, we have a balanced network and storage performance). Figure 2 shows our results where none of the file formats is able to deliver the full hardware performance for reading input files. One third of the performance is already lost in HDFS (maximum throughput 74.9 Gbps out of possible 100 Gbps). The rest of the performance is lost inside the file format implementation, which needs to deal with encoding, buffer and I/O management, compression, etc. The best performer is Apache Arrow which is designed for in-memory columnar datasets. The performance of these file formats are bounded by the performance of the CPU, which is 100% loaded during the experiment. For a detailed analysis of the file formats, please refer to our paper - <a href="https://www.usenix.org/c
onference/atc18/presentation/trivedi">Albis: High-Performance File Format for Big Data Systems (USENIX, ATC’18)</a>.</p>
@@ -132,7 +132,7 @@ This is the first blog post in a multi-part series where we will focus on relati
<li>Careful object materialization using a binary API: To optimize the runtime representation in managed runtimes like the JVM, only objects which are necessary for SQL processing are materialized. Otherwise, a 4 byte integer can be passed around as a byte array (using the binary API of Albis).</li>
</ul>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/core-scalability.svg" width="550" /><figcaption>Figure 3: Core scalability of JSON, Avro, Parquet, ORC, Arrow, and Albis on HDFS/NVMe.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/core-scalability.svg" width="550" /><figcaption>Figure 3: Core scalability of JSON, Avro, Parquet, ORC, Arrow, and Albis on HDFS/NVMe.<p></p></figcaption></div></figure>
<p>Using the Albis format, we revise our previous experiment where we read the input store_sales table from HDFS. In the figure above, we show the performance of Albis and other file formats with number of CPU cores involved. At the right hand of the x-axis, we have performance with all 16 cores engaged, hence, representing the peak possible performance. As evident, Albis delivers 59.9 Gbps out of 74.9 Gbps possible bandwidth with HDFS over NVMe. Albis performance is 1.9 - 21.4x better than other file formats. To give an impression where the performance is coming from, in the table below we show some micro-architectural features for Parquet, ORC, Arrow, and Albis. Our previously discussed design ideas in Albis result in a shorter code path (shown as less instructions required for each row), better cache performance (shows as lower cache misses per row), and clearly better performance (shown as nanoseconds required per row for processing). For a detailed evaluation of Albis please re
fer to our paper.</p>
@@ -171,9 +171,9 @@ This is the first blog post in a multi-part series where we will focus on relati
<h3 id="apache-crail-incubating-with-albis">Apache Crail (Incubating) with Albis</h3>
-<p>For our final experiment, we try to answer the question what it would take to deliver the full 100 Gbps bandwidth for Albis. Certainly, the first bottleneck is to improve the base storage layer performance. Here we use Apache Crail (Incubating) with its <a href="https://en.wikipedia.org/wiki/NVM_Express#NVMeOF">NVMeF</a> storage tier. This tier uses <a href="https://github.com/zrlio/jNVMf">jNVMf library</a> to implement NVMeF stack in Java. As we have shown in a previous blog <a href="http://crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">post</a> that Crail’s NVMeF tier can deliver performance (97.8 Gbps) very close to the hardware limits. Hence, Albis with Crail is a perfect setup to evaluate on high-performance NVMe and RDMA devices. Before we get there, let’s get some calculations right. The store_sales table in the TPC-DS dataset has a data density of 93.9% (out of 100 bytes, only 93.9 is data, others are null values). As we measure the goodput, the e
xpected performance of Albis on Crail is 93.9% of 97.8 Gbps, which calculates to 91.8 Gbps. In our experiments, Albis on Crail delivers 85.5 Gbps. Figure 4 shows more detailed results.</p>
+<p>For our final experiment, we try to answer the question what it would take to deliver the full 100 Gbps bandwidth for Albis. Certainly, the first bottleneck is to improve the base storage layer performance. Here we use Apache Crail (Incubating) with its <a href="https://en.wikipedia.org/wiki/NVM_Express#NVMeOF">NVMeF</a> storage tier. This tier uses <a href="https://github.com/zrlio/jNVMf">jNVMf library</a> to implement NVMeF stack in Java. As we have shown in a previous blog <a href="//crail.incubator.apache.org/blog/2017/08/crail-nvme-fabrics-v1.html">post</a> that Crail’s NVMeF tier can deliver performance (97.8 Gbps) very close to the hardware limits. Hence, Albis with Crail is a perfect setup to evaluate on high-performance NVMe and RDMA devices. Before we get there, let’s get some calculations right. The store_sales table in the TPC-DS dataset has a data density of 93.9% (out of 100 bytes, only 93.9 is data, others are null values). As we measure the goodput, the expect
ed performance of Albis on Crail is 93.9% of 97.8 Gbps, which calculates to 91.8 Gbps. In our experiments, Albis on Crail delivers 85.5 Gbps. Figure 4 shows more detailed results.</p>
-<figure><div style="text-align:center"><img src="http://crail.incubator.apache.org/img/blog/sql-p1/albis-crail.svg" width="550" /><figcaption>Figure 4: Performance of Albis on Crail.<p></p></figcaption></div></figure>
+<figure><div style="text-align:center"><img src="//crail.incubator.apache.org/img/blog/sql-p1/albis-crail.svg" width="550" /><figcaption>Figure 4: Performance of Albis on Crail.<p></p></figcaption></div></figure>
<p>The left half of the figure shows the performance scalability of Albis on Crail in a setup with 1 core (8.9 Gbps) to 16 cores (85.5 Gbps). In comparison, the right half of the figure shows the performance of Crail on HDFS/NVMe at 59.9 Gbps, and on Crail/NVMe at 85.5 Gbps. The last bar shows the performance of Albis if the benchmark does not materialize Java object values. In this configuration, Albis on Crail delivers 91.3 Gbps, which is very close to the expected peak of 91.8 Gbps.</p>
@@ -227,6 +227,6 @@ s.setAttribute('data-timestamp', +new Date());
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/index.html
----------------------------------------------------------------------
diff --git a/content/blog/index.html b/content/blog/index.html
index a61447b..feb6f7c 100644
--- a/content/blog/index.html
+++ b/content/blog/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Latest posts</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/blog-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li class="active">
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -167,6 +167,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/page2/index.html
----------------------------------------------------------------------
diff --git a/content/blog/page2/index.html b/content/blog/page2/index.html
index f0f630a..fe39c56 100644
--- a/content/blog/page2/index.html
+++ b/content/blog/page2/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Latest posts</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/page2-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/page2-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/page2-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/page2-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -171,6 +171,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/page3/index.html
----------------------------------------------------------------------
diff --git a/content/blog/page3/index.html b/content/blog/page3/index.html
index 2a39040..66c1c19 100644
--- a/content/blog/page3/index.html
+++ b/content/blog/page3/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Latest posts</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/page3-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/page3-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/page3-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/page3-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -171,6 +171,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>
http://git-wip-us.apache.org/repos/asf/incubator-crail-website/blob/5b208cb9/content/blog/page4/index.html
----------------------------------------------------------------------
diff --git a/content/blog/page4/index.html b/content/blog/page4/index.html
index 6fadfc3..bd8bfff 100644
--- a/content/blog/page4/index.html
+++ b/content/blog/page4/index.html
@@ -4,13 +4,13 @@
<meta charset="utf-8">
<title>The Apache Crail (Incubating) Project: Latest posts</title>
<meta name="viewport" content="width=device-width, initial-scale=1.0">
- <link href="http://crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
- <link href="http://crail.incubator.apache.org/css/group.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/bootstrap.min.css" rel="stylesheet">
+ <link href="//crail.incubator.apache.org/css/group.css" rel="stylesheet">
<link rel="alternate" type="application/atom+xml" title="Atom"
- href="http://crail.incubator.apache.org/blog/blog.xml">
+ href="//crail.incubator.apache.org/blog/blog.xml">
- <meta property="og:image" content="http://crail.incubator.apache.org/img/blog/preview/page4-summary.png" />
- <meta property="og:image:secure_url" content="http://crail.incubator.apache.org/img/blog/preview/page4-summary.png" />
+ <meta property="og:image" content="//crail.incubator.apache.org/img/blog/preview/page4-summary.png" />
+ <meta property="og:image:secure_url" content="//crail.incubator.apache.org/img/blog/preview/page4-summary.png" />
</head>
<body>
@@ -21,50 +21,50 @@
<li >
- <a href="http://crail.incubator.apache.org/">
+ <a href="//crail.incubator.apache.org/">
Home
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/overview/">
+ <a href="//crail.incubator.apache.org/overview/">
Overview
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/download/">
+ <a href="//crail.incubator.apache.org/download/">
Downloads
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/blog/">
+ <a href="//crail.incubator.apache.org/blog/">
Blog
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/community/">
+ <a href="//crail.incubator.apache.org/community/">
Community
</a>
</li>
<li >
- <a href="http://crail.incubator.apache.org/documentation/">
+ <a href="//crail.incubator.apache.org/documentation/">
Documentation
</a>
</li>
</ul>
- <a href="http://crail.incubator.apache.org/">
- <img src="http://crail.incubator.apache.org/img/crail_logo.png"
- srcset="http://crail.incubator.apache.org/img/crail_logo.png"
+ <a href="//crail.incubator.apache.org/">
+ <img src="//crail.incubator.apache.org/img/crail_logo.png"
+ srcset="//crail.incubator.apache.org/img/crail_logo.png"
alt="Crail" id="logo">
</a>
</div>
@@ -171,6 +171,6 @@
<!-- Support retina images. -->
<script type="text/javascript"
- src="http://crail.incubator.apache.org/js/srcset-polyfill.js"></script>
+ src="//crail.incubator.apache.org/js/srcset-polyfill.js"></script>
</body>
</html>