You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by kt...@apache.org on 2019/09/18 21:45:36 UTC
[accumulo-website] branch asf-site updated: Jekyll build from
master:54c5b50
This is an automated email from the ASF dual-hosted git repository.
kturner pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/accumulo-website.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 6067e9c Jekyll build from master:54c5b50
6067e9c is described below
commit 6067e9c4699ab822342cdddbb3496581c9565dc1
Author: Keith Turner <kt...@apache.org>
AuthorDate: Wed Sep 18 17:45:03 2019 -0400
Jekyll build from master:54c5b50
add a blog post about using HDFS erasure coding with Accumulo (#194)
---
blog/2019/09/17/erasure-coding.html | 485 +++++++++++++++++++++++++++++++
feed.xml | 348 ++++++++++++++++++++--
images/blog/201909_ec/ec-latency-14.png | Bin 0 -> 751396 bytes
images/blog/201909_ec/ec-latency-14e.png | Bin 0 -> 843379 bytes
images/blog/201909_ec/ec-latency-16.png | Bin 0 -> 854230 bytes
index.html | 14 +-
news/index.html | 7 +
search_data.json | 8 +
8 files changed, 833 insertions(+), 29 deletions(-)
diff --git a/blog/2019/09/17/erasure-coding.html b/blog/2019/09/17/erasure-coding.html
new file mode 100644
index 0000000..1556166
--- /dev/null
+++ b/blog/2019/09/17/erasure-coding.html
@@ -0,0 +1,485 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<meta charset="utf-8">
+<meta http-equiv="X-UA-Compatible" content="IE=edge">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<link href="https://maxcdn.bootstrapcdn.com/bootswatch/3.3.7/paper/bootstrap.min.css" rel="stylesheet" integrity="sha384-awusxf8AUojygHf2+joICySzB780jVvQaVCAt1clU3QsyAitLGul28Qxb2r1e5g+" crossorigin="anonymous">
+<link href="//netdna.bootstrapcdn.com/font-awesome/4.0.3/css/font-awesome.css" rel="stylesheet">
+<link rel="stylesheet" type="text/css" href="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.css">
+<link href="/css/accumulo.css" rel="stylesheet" type="text/css">
+
+<title>Using HDFS Erasure Coding with Accumulo</title>
+
+<script src="https://cdnjs.cloudflare.com/ajax/libs/jquery/2.2.4/jquery.min.js" integrity="sha256-BbhdlvQf/xTY9gja0Dq3HiwQF8LaCRTXxZKRutelT44=" crossorigin="anonymous"></script>
+<script src="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/js/bootstrap.min.js" integrity="sha384-Tc5IQib027qvyjSMfHjOMaLkfuWVxZxUPnCJA7l2mCWNIpG9mGCD8wGNIcPD7Txa" crossorigin="anonymous"></script>
+<script type="text/javascript" src="https://cdn.datatables.net/v/bs/jq-2.2.3/dt-1.10.12/datatables.min.js"></script>
+<script>
+ // show location of canonical site if not currently on the canonical site
+ $(function() {
+ var host = window.location.host;
+ if (typeof host !== 'undefined' && host !== 'accumulo.apache.org') {
+ $('#non-canonical').show();
+ }
+ });
+
+ $(function() {
+ // decorate section headers with anchors
+ return $("h2, h3, h4, h5, h6").each(function(i, el) {
+ var $el, icon, id;
+ $el = $(el);
+ id = $el.attr('id');
+ icon = '<i class="fa fa-link"></i>';
+ if (id) {
+ return $el.append($("<a />").addClass("header-link").attr("href", "#" + id).html(icon));
+ }
+ });
+ });
+
+ // fix sidebar width in documentation
+ $(function() {
+ var $affixElement = $('div[data-spy="affix"]');
+ $affixElement.width($affixElement.parent().width());
+ });
+</script>
+
+</head>
+<body style="padding-top: 100px">
+
+ <nav class="navbar navbar-default navbar-fixed-top">
+ <div class="container">
+ <div class="navbar-header">
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target="#navbar-items">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+ <a href="/"><img id="nav-logo" alt="Apache Accumulo" class="img-responsive" src="/images/accumulo-logo.png" width="200"
+ /></a>
+ </div>
+ <div class="collapse navbar-collapse" id="navbar-items">
+ <ul class="nav navbar-nav">
+ <li class="nav-link"><a href="/downloads">Download</a></li>
+ <li class="nav-link"><a href="/tour">Tour</a></li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Releases<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/release/accumulo-2.0.0/">2.0.0 (Latest)</a></li>
+ <li><a href="/release/accumulo-1.9.3/">1.9.3</a></li>
+ <li><a href="/release/">Archive</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Documentation<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/docs/2.x">User Manual (2.x)</a></li>
+ <li><a href="/quickstart-1.x">Quickstart (1.x)</a></li>
+ <li><a href="/accumulo2-maven-plugin">Accumulo Maven Plugin</a></li>
+ <li><a href="/1.9/accumulo_user_manual.html">User Manual (1.9)</a></li>
+ <li><a href="/1.9/apidocs">Javadocs (1.9)</a></li>
+ <li><a href="/external-docs">External Docs</a></li>
+ <li><a href="/docs-archive/">Archive</a></li>
+ </ul>
+ </li>
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#">Community<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="/contact-us">Contact Us</a></li>
+ <li><a href="/how-to-contribute">How To Contribute</a></li>
+ <li><a href="/people">People</a></li>
+ <li><a href="/related-projects">Related Projects</a></li>
+ </ul>
+ </li>
+ <li class="nav-link"><a href="/search">Search</a></li>
+ </ul>
+ <ul class="nav navbar-nav navbar-right">
+ <li class="dropdown">
+ <a class="dropdown-toggle" data-toggle="dropdown" href="#"><img alt="Apache Software Foundation" src="https://www.apache.org/foundation/press/kit/feather.svg" width="15"/><span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li><a href="https://www.apache.org">Apache Homepage <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/licenses/">License <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/sponsorship">Sponsorship <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/security">Security <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/thanks">Thanks <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/foundation/policies/conduct">Code of Conduct <i class="fa fa-external-link"></i></a></li>
+ <li><a href="https://www.apache.org/events/current-event.html">Current Event <i class="fa fa-external-link"></i></a></li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ </div>
+</nav>
+
+
+ <div class="container">
+ <div class="row">
+ <div class="col-md-12">
+
+ <div id="non-canonical" style="display: none; background-color: #F0E68C; padding-left: 1em;">
+ Visit the official site at: <a href="https://accumulo.apache.org">https://accumulo.apache.org</a>
+ </div>
+ <div id="content">
+
+ <h1 class="title">Using HDFS Erasure Coding with Accumulo</h1>
+
+ <p>
+<b>Author: </b> Ed Seidl<br>
+<b>Date: </b> 17 Sep 2019<br>
+
+</p>
+
+<p>HDFS normally stores multiple copies of each file for both performance and durability reasons.
+The number of copies is controlled via HDFS replication settings, and by default is set to 3. Hadoop 3,
+introduced the use of erasure coding (EC), which improves durability while decreasing overhead.
+Since Accumulo 2.0 now supports Hadoop 3, it’s time to take a look at whether using
+EC with Accumulo makes sense.</p>
+
+<ul>
+ <li><a href="#ec-intro">EC Intro</a></li>
+ <li><a href="#ec-performance">EC Performance</a></li>
+ <li><a href="#accumulo-performance-with-ec">Accumulo Performance with EC</a></li>
+</ul>
+
+<h3 id="ec-intro">EC Intro</h3>
+
+<p>By default HDFS achieves durability via block replication. Usually
+the replication count is 3, resulting in a storage overhead of 200%. Hadoop 3
+introduced EC as a better way to achieve durability. More info can be
+found <a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html">here</a>.
+EC behaves much like RAID 5 or 6…for <em>k</em> blocks of data, <em>m</em> blocks of
+parity data are generated, from which the original data can be recovered in the
+event of disk or node failures (erasures, in EC parlance). A typical EC scheme is Reed-Solomon 6-3, where
+6 data blocks produce 3 parity blocks, an overhead of only 50%. In addition
+to doubling the available disk space, RS-6-3 is also more fault
+tolerant…a loss of 3 data blocks can be tolerated, where triple replication
+can only lose two blocks.</p>
+
+<p>More storage, better resiliency, so what’s the catch? One concern is
+the time spent calculating the parity blocks. Unlike replication
+, where a client writes a block, and then the DataNodes replicate
+the data, an EC HDFS client is responsible for computing the parity and sending that
+to the DataNodes. This increases the CPU and network load on the client. The CPU
+hit can be mitigated by using Intels ISA-L library, but only on CPUs
+that support AVX or AVX2 instructions. (See <a href="https://www.slideshare.net/HadoopSummit/debunking-the-myths-of-hdfs-erasure-coding-performance">EC Myths</a> and <a href="https://blog.cloudera.com/introduction-to-hdfs-erasure-coding-in-apache-hadoop/">EC Introduction</a>
+for some interesting claims). In addition, unlike the serial replication I/O path,
+the EC I/O path is parallel providing greater throughput. In our testing, sequential writes to
+an EC directory were as much as 3 times faster than a replication directory
+, and reads were up to 2 times faster.</p>
+
+<p>Another side effect of EC is loss of data locality. For performance reasons, EC
+data blocks are striped, so multiple DataNodes must be contacted to read a single
+block of data. For large sequential reads this is not a
+problem, but it can be an issue for small random lookups. For the latter case,
+using RS 6-3 with 64KB stripes mitigates some of the random lookup pain
+without compromising sequential read/write performance.</p>
+
+<h4 id="important-warning">Important Warning</h4>
+
+<p>Before continuing, an important caveat; the current implementation of EC on Hadoop supports neither hsync
+nor hflush. Both of these operations are silent no-ops (EC <a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html#Limitations">limitations</a>). We discovered this the hard
+way when a data center power loss resulted in write-ahead log corruption, which were
+stored in an EC directory. To avoid this problem ensure all
+WAL directories use replication. It’s probably a good idea to keep the
+accumulo namespace replicated as well, but we have no evidence to back up that assertion. As with all
+things, don’t test on production data.</p>
+
+<h3 id="ec-performance">EC Performance</h3>
+
+<p>To test EC performance, we created a series of clusters on AWS. Our Accumulo stack consisted of
+Hadoop 3.1.1 built with the Intel ISA-L library enabled, Zookeeper 3.4.13, and Accumulo 1.9.3 configured
+to work with Hadoop 3 (we did our testing before the official release of Accumulo 2.0). The encoding
+policy is set per-directory using the <a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html#Administrative_commands">hdfs</a> command-line tool. To set the encoding policy
+for an Accumulo table, first find the table ID (for instance using the Accumulo shell’s
+“table -l” command), and then from the command line set the policy for the corresponding directory
+under /accumulo/tables. Note that changing the policy on a directory will set the policy for
+child directories, but will not change any files contained within. To change the policy on an existing
+Accumulo table, you must first set the encoding policy, and then run a major compaction to rewrite
+the RFiles for the table.</p>
+
+<p>Our first tests were of sequential read and write performance straight to HDFS. For this test we had
+a cluster of 32 HDFS nodes (c5.4xlarge <a href="https://aws.amazon.com/ec2/instance-types/">AWS</a> instances), 16 Spark nodes (r5.4xlarge),
+3 zookeepers (r5.xlarge), and 1 master (r5.2xlarge).</p>
+
+<p>The first table below shows the results for writing a 1TB file. The results are the average of three runs
+for each of the directory encodings Reed-Solomon (RS) 6-3 with 64KB stripes, RS 6-3 with 1MB stripes,
+RS 10-4 with 1MB stripes, and the default triple replication. We also varied the number of concurrent
+Spark executors, performing tests with 16 executors that did not stress the cluster in any area, and with
+128 executors which exhausted our network bandwidth allotment of 5 Gbps. As can be seen, in the 16 executor
+environment, we saw greater than a 3X bump in throughput using RS 10-4 with 1MB stripes over triple replication.
+At saturation, the speed up was still over 2X, which is in line with the results from <a href="https://www.slideshare.net/HadoopSummit/debunking-the-myths-of-hdfs-erasure-coding-performance">EC Myths</a>. Also of note,
+using RS 6-3 with 64KB stripes performed better than the same with 1MB stripes, which is a nice result for Accumulo,
+as we’ll show later.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">16 executors</th>
+ <th style="text-align: right">128 executors</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">2.19 GB/s</td>
+ <td style="text-align: right">4.13 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">6.33 GB/s</td>
+ <td style="text-align: right">8.11 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">6.22 GB/s</td>
+ <td style="text-align: right">7.93 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">7.09 GB/s</td>
+ <td style="text-align: right">8.34 GB/s</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Our read tests are not as dramatic as those in <a href="https://www.slideshare.net/HadoopSummit/debunking-the-myths-of-hdfs-erasure-coding-performance">EC Myths</a>, but still looking good for EC. Here we show the
+results for reading back the 1TB file created in the write test using 16 Spark executors. In addition to
+the straight read tests, we also performed tests with 2 DataNodes disabled to simulate the performance hit
+of failures which require data repair in the foreground. Finally, we tested the read performance
+after a background rebuild of the filesystem. We did this to see if the foreground rebuild or
+the loss of 2 DataNodes was the major contributor to any performance degradation. As can be seen,
+EC read performance is close to 2X faster than replication, even in the face of failures.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">32 nodes<br />no failures</th>
+ <th style="text-align: right">30 nodes<br />with failures</th>
+ <th style="text-align: right">30 nodes<br />no failures</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">3.95 GB/s</td>
+ <td style="text-align: right">3.99 GB/s</td>
+ <td style="text-align: right">3.89 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">7.36 GB/s</td>
+ <td style="text-align: right">7.27 GB/s</td>
+ <td style="text-align: right">7.16 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">6.59 GB/s</td>
+ <td style="text-align: right">6.47 GB/s</td>
+ <td style="text-align: right">6.53 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">6.21 GB/s</td>
+ <td style="text-align: right">6.08 GB/s</td>
+ <td style="text-align: right">6.21 GB/s</td>
+ </tr>
+ </tbody>
+</table>
+
+<h3 id="accumulo-performance-with-ec">Accumulo Performance with EC</h3>
+
+<p>While the above results are impressive, they are not representative of how Accumulo uses HDFS. For starters,
+Accumulo sequential I/O is doing far more than just reading or writing files; compression and serialization,
+for example, place quite a load upon the tablet server CPUs. An example to illustrate this is shown below.
+The time in minutes to bulk-write 400 million rows to RFiles with 40 Spark executors is listed for both EC
+using RS 6-3 with 1MB stripes and triple replication. The choice of compressor has a much more profound
+effect on the write times than the choice of underlying encoding for the directory being written to
+(although without compression EC is much faster than replication).</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Compressor</th>
+ <th style="text-align: right">RS 6-3 1MB</th>
+ <th style="text-align: right">Replication</th>
+ <th style="text-align: right">File size (GB)</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>gz</td>
+ <td style="text-align: right">2.7</td>
+ <td style="text-align: right">2.7</td>
+ <td style="text-align: right">21.3</td>
+ </tr>
+ <tr>
+ <td>none</td>
+ <td style="text-align: right">2.0</td>
+ <td style="text-align: right">3.0</td>
+ <td style="text-align: right">158.5</td>
+ </tr>
+ <tr>
+ <td>snappy</td>
+ <td style="text-align: right">1.6</td>
+ <td style="text-align: right">1.6</td>
+ <td style="text-align: right">38.4</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Of much more importance to Accumulo performance is read latency. A frequent use case for our group is to obtain a
+number of row IDs from an index and then use a BatchScanner to read those individual rows.
+In this use case, the time to access a single row is far more important than the raw I/O performance. To test
+Accumulo’s performance with EC for this use case, we did a series of tests against a 10 billion row table,
+with each row consisting of 10 columns. 16 Spark executors each performed 10000 queries, where each query
+sought 10 random rows. Thus 16 million individual rows were returned in batches of 10. For each batch of
+10, the time in milliseconds was captured, and theses times were collected in a histogram of 50ms buckets, with
+a catch-all bucket for queries that took over 1 second. For this test we reconfigured our cluster to make use
+of c5n.4xlarge nodes featuring must faster networking speeds (15 Gbps sustained vs 5 Gbps for
+c5.4xlarge). Because these nodes are in short supply, we ran with only 16 HDFS nodes (c5n.4xlarge),
+but still had 16 Spark nodes (also c5n.4xlarge). Zookeeper and master nodes remained the same.</p>
+
+<p>In the table below, we show the min, max, and average times in milliseconds for each batch of 10 across
+four different encoding policies. The clear winner here is replication, and the clear loser RS 10-4 with
+1MB stripes, but RS 6-3 with 64KB stripes is not looking too bad.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">Min</th>
+ <th style="text-align: right">Avg</th>
+ <th style="text-align: right">Max</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">40</td>
+ <td style="text-align: right">105</td>
+ <td style="text-align: right">2148</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">30</td>
+ <td style="text-align: right">68</td>
+ <td style="text-align: right">1297</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">23</td>
+ <td style="text-align: right">43</td>
+ <td style="text-align: right">1064</td>
+ </tr>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">11</td>
+ <td style="text-align: right">23</td>
+ <td style="text-align: right">731</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>The above results also hold in the event of errors. The next table shows the same test, but with 2 DataNodes
+disabled to simulate failures that require foreground rebuilds. Again, replication wins, and RS 10-4 1MB
+loses, but RS 6-3 64KB remains a viable option.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">Min</th>
+ <th style="text-align: right">Avg</th>
+ <th style="text-align: right">Max</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">53</td>
+ <td style="text-align: right">143</td>
+ <td style="text-align: right">3221</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">34</td>
+ <td style="text-align: right">113</td>
+ <td style="text-align: right">1662</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">24</td>
+ <td style="text-align: right">61</td>
+ <td style="text-align: right">1402</td>
+ </tr>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">12</td>
+ <td style="text-align: right">26</td>
+ <td style="text-align: right">304</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>The images below show a plots of the histograms. The third plot was generated with 14 HDFS DataNodes, but after
+all missing data had been repaired. Again, this was done to see how much of the performance degradation could be
+attributed to missing data, and how much to simply having less computing power available.</p>
+
+<center>
+<img src="/images/blog/201909_ec/ec-latency-16.png" width="75%" /><br /><br />
+
+<img src="/images/blog/201909_ec/ec-latency-14e.png" width="75%" /><br /><br />
+
+<img src="/images/blog/201909_ec/ec-latency-14.png" width="75%" />
+</center>
+
+<h3 id="conclusion">Conclusion</h3>
+<p>HDFS with erasure coding has the potential to double your available Accumulo storage, at the cost of a hit in
+random seek times, but a potential increase in sequential scan performance. We will be proposing some changes
+to Accumulo to make working with EC a bit easier. Our initial thoughts are collected in this
+Accumulo dev list <a href="https://lists.apache.org/thread.html/4ac5b0f664e15fa120e748892612f1e417b7dee3e1539669d179900c@%3Cdev.accumulo.apache.org%3E">post</a>.</p>
+
+
+
+<p><strong>View all posts in the <a href="/news">news archive</a></strong></p>
+
+ </div>
+
+
+<footer>
+
+ <p><a href="https://www.apache.org/foundation/contributing"><img src="https://www.apache.org/images/SupportApache-small.png" alt="Support the ASF" id="asf-logo" height="100" /></a></p>
+
+ <p>Copyright © 2011-2019 <a href="https://www.apache.org">The Apache Software Foundation</a>.
+Licensed under the <a href="https://www.apache.org/licenses/">Apache License, Version 2.0</a>.</p>
+
+ <p>Apache®, the names of Apache projects and their logos, and the multicolor feather
+logo are registered trademarks or trademarks of The Apache Software Foundation
+in the United States and/or other countries.</p>
+
+</footer>
+
+
+ </div>
+ </div>
+ </div>
+</body>
+</html>
diff --git a/feed.xml b/feed.xml
index 3a37d3e..2e6fead 100644
--- a/feed.xml
+++ b/feed.xml
@@ -6,12 +6,336 @@
</description>
<link>https://accumulo.apache.org/</link>
<atom:link href="https://accumulo.apache.org/feed.xml" rel="self" type="application/rss+xml"/>
- <pubDate>Wed, 11 Sep 2019 15:34:41 -0400</pubDate>
- <lastBuildDate>Wed, 11 Sep 2019 15:34:41 -0400</lastBuildDate>
+ <pubDate>Wed, 18 Sep 2019 17:44:54 -0400</pubDate>
+ <lastBuildDate>Wed, 18 Sep 2019 17:44:54 -0400</lastBuildDate>
<generator>Jekyll v3.8.6</generator>
<item>
+ <title>Using HDFS Erasure Coding with Accumulo</title>
+ <description><p>HDFS normally stores multiple copies of each file for both performance and durability reasons.
+The number of copies is controlled via HDFS replication settings, and by default is set to 3. Hadoop 3,
+introduced the use of erasure coding (EC), which improves durability while decreasing overhead.
+Since Accumulo 2.0 now supports Hadoop 3, it’s time to take a look at whether using
+EC with Accumulo makes sense.</p>
+
+<ul>
+ <li><a href="#ec-intro">EC Intro</a></li>
+ <li><a href="#ec-performance">EC Performance</a></li>
+ <li><a href="#accumulo-performance-with-ec">Accumulo Performance with EC</a></li>
+</ul>
+
+<h3 id="ec-intro">EC Intro</h3>
+
+<p>By default HDFS achieves durability via block replication. Usually
+the replication count is 3, resulting in a storage overhead of 200%. Hadoop 3
+introduced EC as a better way to achieve durability. More info can be
+found <a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html">here</a>.
+EC behaves much like RAID 5 or 6…for <em>k</em> blocks of data, <em>m</em> blocks of
+parity data are generated, from which the original data can be recovered in the
+event of disk or node failures (erasures, in EC parlance). A typical EC scheme is Reed-Solomon 6-3, where
+6 data blocks produce 3 parity blocks, an overhead of only 50%. In addition
+to doubling the available disk space, RS-6-3 is also more fault
+tolerant…a loss of 3 data blocks can be tolerated, where triple replication
+can only lose two blocks.</p>
+
+<p>More storage, better resiliency, so what’s the catch? One concern is
+the time spent calculating the parity blocks. Unlike replication
+, where a client writes a block, and then the DataNodes replicate
+the data, an EC HDFS client is responsible for computing the parity and sending that
+to the DataNodes. This increases the CPU and network load on the client. The CPU
+hit can be mitigated by using Intels ISA-L library, but only on CPUs
+that support AVX or AVX2 instructions. (See <a href="https://www.slideshare.net/HadoopSummit/debunking-the-myths-of-hdfs-erasure-coding-performance">EC Myths</a> and <a href="https://blog.cloudera.com/introduction-to-hdfs-erasure-coding-in-apache-hadoop/">EC Introduction</a>
+for some interesting claims). In addition, unlike the serial replication I/O path,
+the EC I/O path is parallel providing greater throughput. In our testing, sequential writes to
+an EC directory were as much as 3 times faster than a replication directory
+, and reads were up to 2 times faster.</p>
+
+<p>Another side effect of EC is loss of data locality. For performance reasons, EC
+data blocks are striped, so multiple DataNodes must be contacted to read a single
+block of data. For large sequential reads this is not a
+problem, but it can be an issue for small random lookups. For the latter case,
+using RS 6-3 with 64KB stripes mitigates some of the random lookup pain
+without compromising sequential read/write performance.</p>
+
+<h4 id="important-warning">Important Warning</h4>
+
+<p>Before continuing, an important caveat; the current implementation of EC on Hadoop supports neither hsync
+nor hflush. Both of these operations are silent no-ops (EC <a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html#Limitations">limitations</a>). We discovered this the hard
+way when a data center power loss resulted in write-ahead log corruption, which were
+stored in an EC directory. To avoid this problem ensure all
+WAL directories use replication. It’s probably a good idea to keep the
+accumulo namespace replicated as well, but we have no evidence to back up that assertion. As with all
+things, don’t test on production data.</p>
+
+<h3 id="ec-performance">EC Performance</h3>
+
+<p>To test EC performance, we created a series of clusters on AWS. Our Accumulo stack consisted of
+Hadoop 3.1.1 built with the Intel ISA-L library enabled, Zookeeper 3.4.13, and Accumulo 1.9.3 configured
+to work with Hadoop 3 (we did our testing before the official release of Accumulo 2.0). The encoding
+policy is set per-directory using the <a href="https://hadoop.apache.org/docs/r3.2.0/hadoop-project-dist/hadoop-hdfs/HDFSErasureCoding.html#Administrative_commands">hdfs</a> command-line tool. To set the encoding policy
+for an Accumulo table, first find the table ID (for instance using the Accumulo shell’s
+“table -l” command), and then from the command line set the policy for the corresponding directory
+under /accumulo/tables. Note that changing the policy on a directory will set the policy for
+child directories, but will not change any files contained within. To change the policy on an existing
+Accumulo table, you must first set the encoding policy, and then run a major compaction to rewrite
+the RFiles for the table.</p>
+
+<p>Our first tests were of sequential read and write performance straight to HDFS. For this test we had
+a cluster of 32 HDFS nodes (c5.4xlarge <a href="https://aws.amazon.com/ec2/instance-types/">AWS</a> instances), 16 Spark nodes (r5.4xlarge),
+3 zookeepers (r5.xlarge), and 1 master (r5.2xlarge).</p>
+
+<p>The first table below shows the results for writing a 1TB file. The results are the average of three runs
+for each of the directory encodings Reed-Solomon (RS) 6-3 with 64KB stripes, RS 6-3 with 1MB stripes,
+RS 10-4 with 1MB stripes, and the default triple replication. We also varied the number of concurrent
+Spark executors, performing tests with 16 executors that did not stress the cluster in any area, and with
+128 executors which exhausted our network bandwidth allotment of 5 Gbps. As can be seen, in the 16 executor
+environment, we saw greater than a 3X bump in throughput using RS 10-4 with 1MB stripes over triple replication.
+At saturation, the speed up was still over 2X, which is in line with the results from <a href="https://www.slideshare.net/HadoopSummit/debunking-the-myths-of-hdfs-erasure-coding-performance">EC Myths</a>. Also of note,
+using RS 6-3 with 64KB stripes performed better than the same with 1MB stripes, which is a nice result for Accumulo,
+as we’ll show later.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">16 executors</th>
+ <th style="text-align: right">128 executors</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">2.19 GB/s</td>
+ <td style="text-align: right">4.13 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">6.33 GB/s</td>
+ <td style="text-align: right">8.11 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">6.22 GB/s</td>
+ <td style="text-align: right">7.93 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">7.09 GB/s</td>
+ <td style="text-align: right">8.34 GB/s</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Our read tests are not as dramatic as those in <a href="https://www.slideshare.net/HadoopSummit/debunking-the-myths-of-hdfs-erasure-coding-performance">EC Myths</a>, but still looking good for EC. Here we show the
+results for reading back the 1TB file created in the write test using 16 Spark executors. In addition to
+the straight read tests, we also performed tests with 2 DataNodes disabled to simulate the performance hit
+of failures which require data repair in the foreground. Finally, we tested the read performance
+after a background rebuild of the filesystem. We did this to see if the foreground rebuild or
+the loss of 2 DataNodes was the major contributor to any performance degradation. As can be seen,
+EC read performance is close to 2X faster than replication, even in the face of failures.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">32 nodes<br />no failures</th>
+ <th style="text-align: right">30 nodes<br />with failures</th>
+ <th style="text-align: right">30 nodes<br />no failures</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">3.95 GB/s</td>
+ <td style="text-align: right">3.99 GB/s</td>
+ <td style="text-align: right">3.89 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">7.36 GB/s</td>
+ <td style="text-align: right">7.27 GB/s</td>
+ <td style="text-align: right">7.16 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">6.59 GB/s</td>
+ <td style="text-align: right">6.47 GB/s</td>
+ <td style="text-align: right">6.53 GB/s</td>
+ </tr>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">6.21 GB/s</td>
+ <td style="text-align: right">6.08 GB/s</td>
+ <td style="text-align: right">6.21 GB/s</td>
+ </tr>
+ </tbody>
+</table>
+
+<h3 id="accumulo-performance-with-ec">Accumulo Performance with EC</h3>
+
+<p>While the above results are impressive, they are not representative of how Accumulo uses HDFS. For starters,
+Accumulo sequential I/O is doing far more than just reading or writing files; compression and serialization,
+for example, place quite a load upon the tablet server CPUs. An example to illustrate this is shown below.
+The time in minutes to bulk-write 400 million rows to RFiles with 40 Spark executors is listed for both EC
+using RS 6-3 with 1MB stripes and triple replication. The choice of compressor has a much more profound
+effect on the write times than the choice of underlying encoding for the directory being written to
+(although without compression EC is much faster than replication).</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Compressor</th>
+ <th style="text-align: right">RS 6-3 1MB</th>
+ <th style="text-align: right">Replication</th>
+ <th style="text-align: right">File size (GB)</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>gz</td>
+ <td style="text-align: right">2.7</td>
+ <td style="text-align: right">2.7</td>
+ <td style="text-align: right">21.3</td>
+ </tr>
+ <tr>
+ <td>none</td>
+ <td style="text-align: right">2.0</td>
+ <td style="text-align: right">3.0</td>
+ <td style="text-align: right">158.5</td>
+ </tr>
+ <tr>
+ <td>snappy</td>
+ <td style="text-align: right">1.6</td>
+ <td style="text-align: right">1.6</td>
+ <td style="text-align: right">38.4</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Of much more importance to Accumulo performance is read latency. A frequent use case for our group is to obtain a
+number of row IDs from an index and then use a BatchScanner to read those individual rows.
+In this use case, the time to access a single row is far more important than the raw I/O performance. To test
+Accumulo’s performance with EC for this use case, we did a series of tests against a 10 billion row table,
+with each row consisting of 10 columns. 16 Spark executors each performed 10000 queries, where each query
+sought 10 random rows. Thus 16 million individual rows were returned in batches of 10. For each batch of
+10, the time in milliseconds was captured, and theses times were collected in a histogram of 50ms buckets, with
+a catch-all bucket for queries that took over 1 second. For this test we reconfigured our cluster to make use
+of c5n.4xlarge nodes featuring must faster networking speeds (15 Gbps sustained vs 5 Gbps for
+c5.4xlarge). Because these nodes are in short supply, we ran with only 16 HDFS nodes (c5n.4xlarge),
+but still had 16 Spark nodes (also c5n.4xlarge). Zookeeper and master nodes remained the same.</p>
+
+<p>In the table below, we show the min, max, and average times in milliseconds for each batch of 10 across
+four different encoding policies. The clear winner here is replication, and the clear loser RS 10-4 with
+1MB stripes, but RS 6-3 with 64KB stripes is not looking too bad.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">Min</th>
+ <th style="text-align: right">Avg</th>
+ <th style="text-align: right">Max</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">40</td>
+ <td style="text-align: right">105</td>
+ <td style="text-align: right">2148</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">30</td>
+ <td style="text-align: right">68</td>
+ <td style="text-align: right">1297</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">23</td>
+ <td style="text-align: right">43</td>
+ <td style="text-align: right">1064</td>
+ </tr>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">11</td>
+ <td style="text-align: right">23</td>
+ <td style="text-align: right">731</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>The above results also hold in the event of errors. The next table shows the same test, but with 2 DataNodes
+disabled to simulate failures that require foreground rebuilds. Again, replication wins, and RS 10-4 1MB
+loses, but RS 6-3 64KB remains a viable option.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>Encoding</th>
+ <th style="text-align: right">Min</th>
+ <th style="text-align: right">Avg</th>
+ <th style="text-align: right">Max</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>RS 10-4 1MB</td>
+ <td style="text-align: right">53</td>
+ <td style="text-align: right">143</td>
+ <td style="text-align: right">3221</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 1MB</td>
+ <td style="text-align: right">34</td>
+ <td style="text-align: right">113</td>
+ <td style="text-align: right">1662</td>
+ </tr>
+ <tr>
+ <td>RS 6-3 64KB</td>
+ <td style="text-align: right">24</td>
+ <td style="text-align: right">61</td>
+ <td style="text-align: right">1402</td>
+ </tr>
+ <tr>
+ <td>Replication</td>
+ <td style="text-align: right">12</td>
+ <td style="text-align: right">26</td>
+ <td style="text-align: right">304</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>The images below show a plots of the histograms. The third plot was generated with 14 HDFS DataNodes, but after
+all missing data had been repaired. Again, this was done to see how much of the performance degradation could be
+attributed to missing data, and how much to simply having less computing power available.</p>
+
+<center>
+<img src="/images/blog/201909_ec/ec-latency-16.png" width="75%" /><br /><br />
+
+<img src="/images/blog/201909_ec/ec-latency-14e.png" width="75%" /><br /><br />
+
+<img src="/images/blog/201909_ec/ec-latency-14.png" width="75%" />
+</center>
+
+<h3 id="conclusion">Conclusion</h3>
+<p>HDFS with erasure coding has the potential to double your available Accumulo storage, at the cost of a hit in
+random seek times, but a potential increase in sequential scan performance. We will be proposing some changes
+to Accumulo to make working with EC a bit easier. Our initial thoughts are collected in this
+Accumulo dev list <a href="https://lists.apache.org/thread.html/4ac5b0f664e15fa120e748892612f1e417b7dee3e1539669d179900c@%3Cdev.accumulo.apache.org%3E">post</a>.</p>
+
+</description>
+ <pubDate>Tue, 17 Sep 2019 00:00:00 -0400</pubDate>
+ <link>https://accumulo.apache.org/blog/2019/09/17/erasure-coding.html</link>
+ <guid isPermaLink="true">https://accumulo.apache.org/blog/2019/09/17/erasure-coding.html</guid>
+
+
+ <category>blog</category>
+
+ </item>
+
+ <item>
<title>Using S3 as a data store for Accumulo</title>
<description><p>Accumulo can store its files in S3, however S3 does not support the needs of
write ahead logs and the Accumulo metadata table. One way to solve this problem
@@ -900,25 +1224,5 @@ flushed, compacted, scanned, and deleted the table.</li>
</item>
- <item>
- <title>Accumulo Summit is on October 15th!</title>
- <description><p>The <a href="http://accumulosummit.com/">Fifth Annual Accumulo Summit</a> will be held on October 15, 2018 at the <a href="http://accumulosummit.com/about/venue/">Sheraton Columbia Town Center Hotel</a> in Columbia, MD.</p>
-
-<p>This day-long event offers a unique opportunity for attendees to get introduced to Apache Accumulo, sharpen their skillsets, and connect with leading Accumulo users and developers.</p>
-
-<p>Have a great idea you’d like to share? Engineers, architects, and business leaders are encouraged to share their experiences or present a topic that would be of interest to the Accumulo community. <a href="http://accumulosummit.com/program/submit-talk/">Talks can be submitted </a> through August 1st.</p>
-
-<p><a href="http://accumulosummit.com/register/">Early bird registration is now open</a>! Sign up before September 1st to save $50 off the regular admission price.</p>
-
-</description>
- <pubDate>Thu, 05 Jul 2018 00:00:00 -0400</pubDate>
- <link>https://accumulo.apache.org/blog/2018/07/05/accumulo-summit.html</link>
- <guid isPermaLink="true">https://accumulo.apache.org/blog/2018/07/05/accumulo-summit.html</guid>
-
-
- <category>blog</category>
-
- </item>
-
</channel>
</rss>
diff --git a/images/blog/201909_ec/ec-latency-14.png b/images/blog/201909_ec/ec-latency-14.png
new file mode 100644
index 0000000..a4db326
Binary files /dev/null and b/images/blog/201909_ec/ec-latency-14.png differ
diff --git a/images/blog/201909_ec/ec-latency-14e.png b/images/blog/201909_ec/ec-latency-14e.png
new file mode 100644
index 0000000..ade951f
Binary files /dev/null and b/images/blog/201909_ec/ec-latency-14e.png differ
diff --git a/images/blog/201909_ec/ec-latency-16.png b/images/blog/201909_ec/ec-latency-16.png
new file mode 100644
index 0000000..fef02d2
Binary files /dev/null and b/images/blog/201909_ec/ec-latency-16.png differ
diff --git a/index.html b/index.html
index d6538e1..303d3ec 100644
--- a/index.html
+++ b/index.html
@@ -179,35 +179,35 @@
<div class="row latest-news-item">
<div class="col-sm-12" style="margin-bottom: 5px">
<span style="font-size: 12px; margin-right: 5px;">Sep 2019</span>
- <a href="/blog/2019/09/10/accumulo-S3-notes.html">Using S3 as a data store for Accumulo</a>
+ <a href="/blog/2019/09/17/erasure-coding.html">Using HDFS Erasure Coding with Accumulo</a>
</div>
</div>
<div class="row latest-news-item">
<div class="col-sm-12" style="margin-bottom: 5px">
- <span style="font-size: 12px; margin-right: 5px;">Aug 2019</span>
- <a href="/blog/2019/08/12/why-upgrade.html">Top 10 Reasons to Upgrade</a>
+ <span style="font-size: 12px; margin-right: 5px;">Sep 2019</span>
+ <a href="/blog/2019/09/10/accumulo-S3-notes.html">Using S3 as a data store for Accumulo</a>
</div>
</div>
<div class="row latest-news-item">
<div class="col-sm-12" style="margin-bottom: 5px">
<span style="font-size: 12px; margin-right: 5px;">Aug 2019</span>
- <a href="/release/accumulo-2.0.0/">Apache Accumulo 2.0.0</a>
+ <a href="/blog/2019/08/12/why-upgrade.html">Top 10 Reasons to Upgrade</a>
</div>
</div>
<div class="row latest-news-item">
<div class="col-sm-12" style="margin-bottom: 5px">
- <span style="font-size: 12px; margin-right: 5px;">Apr 2019</span>
- <a href="/blog/2019/04/24/using-spark-with-accumulo.html">Using Apache Spark with Accumulo</a>
+ <span style="font-size: 12px; margin-right: 5px;">Aug 2019</span>
+ <a href="/release/accumulo-2.0.0/">Apache Accumulo 2.0.0</a>
</div>
</div>
<div class="row latest-news-item">
<div class="col-sm-12" style="margin-bottom: 5px">
<span style="font-size: 12px; margin-right: 5px;">Apr 2019</span>
- <a href="/release/accumulo-1.9.3/">Apache Accumulo 1.9.3</a>
+ <a href="/blog/2019/04/24/using-spark-with-accumulo.html">Using Apache Spark with Accumulo</a>
</div>
</div>
diff --git a/news/index.html b/news/index.html
index b186191..f63e7b1 100644
--- a/news/index.html
+++ b/news/index.html
@@ -147,6 +147,13 @@
<div class="row" style="margin-top: 15px">
+ <div class="col-md-1">Sep 17</div>
+ <div class="col-md-10"><a href="/blog/2019/09/17/erasure-coding.html">Using HDFS Erasure Coding with Accumulo</a></div>
+ </div>
+
+
+
+ <div class="row" style="margin-top: 15px">
<div class="col-md-1">Sep 10</div>
<div class="col-md-10"><a href="/blog/2019/09/10/accumulo-S3-notes.html">Using S3 as a data store for Accumulo</a></div>
</div>
diff --git a/search_data.json b/search_data.json
index 7a183d4..919626b 100644
--- a/search_data.json
+++ b/search_data.json
@@ -295,6 +295,14 @@
},
+ "blog-2019-09-17-erasure-coding-html": {
+ "title": "Using HDFS Erasure Coding with Accumulo",
+ "content" : "HDFS normally stores multiple copies of each file for both performance and durability reasons. The number of copies is controlled via HDFS replication settings, and by default is set to 3. Hadoop 3, introduced the use of erasure coding (EC), which improves durability while decreasing overhead.Since Accumulo 2.0 now supports Hadoop 3, it’s time to take a look at whether usingEC with Accumulo makes sense. EC Intro EC Performance Accumulo Performance with ECEC IntroBy [...]
+ "url": " /blog/2019/09/17/erasure-coding.html",
+ "categories": "blog"
+ }
+ ,
+
"blog-2019-09-10-accumulo-s3-notes-html": {
"title": "Using S3 as a data store for Accumulo",
"content" : "Accumulo can store its files in S3, however S3 does not support the needs ofwrite ahead logs and the Accumulo metadata table. One way to solve this problemis to store the metadata table and write ahead logs in HDFS and everything elsein S3. This post shows how to do that using Accumulo 2.0 and Hadoop 3.2.0.Running on S3 requires a new feature in Accumulo 2.0, that volume choosers areaware of write ahead logs.Hadoop setupAt least the following settings should be added [...]