You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2016/06/11 18:15:53 UTC
[1/6] orc git commit: Pushing update for ORC-65
Repository: orc
Updated Branches:
refs/heads/asf-site 6ef32bc20 -> 1eb37b727
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/2016/01/25/ORC-1.0.0/index.html
----------------------------------------------------------------------
diff --git a/news/2016/01/25/ORC-1.0.0/index.html b/news/2016/01/25/ORC-1.0.0/index.html
index adaedc4..fd177c7 100644
--- a/news/2016/01/25/ORC-1.0.0/index.html
+++ b/news/2016/01/25/ORC-1.0.0/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -131,9 +133,9 @@ contains the native C++ ORC reader and some tools.</p>
<p>Release Artifacts:</p>
<ul>
- <li>Source: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz">orc-1.0.0.tgz</a></li>
+ <li>Source: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.0.0/orc-1.0.0.tgz">orc-1.0.0.tgz</a></li>
<li>Git Tag:
-<a href="https://github.com/apache/orc/releases/tag/rel%2Frelease-1.0.0">rel/release-1.0.0 (704d97e)</a></li>
+<a href="https://github.com/apache/orc/releases/tag/rel/release-1.0.0">rel/release-1.0.0 (704d97e)</a></li>
<li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz.asc">GPG Signature</a>
signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
<li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz.mds">8AD5111F 0CA3B72F</a></li>
@@ -186,6 +188,10 @@ reader.</li>
<h4>Recent Releases</h4>
<ul>
+ <li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
<li class="current">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -197,6 +203,8 @@ reader.</li>
+
+
<li class="">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/2016/06/10/ORC-1.1.0/index.html
----------------------------------------------------------------------
diff --git a/news/2016/06/10/ORC-1.1.0/index.html b/news/2016/06/10/ORC-1.1.0/index.html
new file mode 100644
index 0000000..f958aa4
--- /dev/null
+++ b/news/2016/06/10/ORC-1.1.0/index.html
@@ -0,0 +1,273 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>ORC 1.1.0 Released</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="current">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="current">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="news">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the blog\u2026</option>
+ <option value="/news/">Home</option>
+ <optgroup label="v1.x">
+
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
+ <option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
+
+ <option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
+
+ <option value="/news/2015/06/26/new-logo/">ORC adopts new logo</option>
+
+ <option value="/news/2015/05/11/new-committers/">ORC adds 7 committers</option>
+
+ <option value="/news/2015/04/22/orc-as-tlp/">ORC becomes an Apache Top Level Project</option>
+
+ </optgroup>
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h2>
+ ORC 1.1.0 Released
+ <a href="/news/2016/06/10/ORC-1.1.0/" class="permalink" title="Permalink">\u221e</a>
+ </h2>
+ <span class="post-category">
+ <span class="label">
+ release
+ </span>
+ </span>
+ <div class="post-meta">
+ <span class="post-date">
+ 10 Jun 2016
+ </span>
+ <a href="https://people.apache.org/~omalley" class="post-author">
+ <img src="https://people.apache.org/~omalley/omalley.png"
+ class="avatar" alt="omalley avatar"
+ width="24" height="24">
+ omalley
+ </a>
+ </div>
+ <div class="post-content">
+ <p>The ORC team is excited to announce the release of ORC v1.1.0. This release
+contains the Java reader and writer and the native C++ ORC reader and tools.</p>
+
+<p>Release Artifacts:</p>
+
+<ul>
+ <li>Released: 10 Jun 2016</li>
+ <li>Source code: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.1.0/orc-1.1.0.tgz">orc-1.1.0.tgz</a></li>
+ <li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.asc">GPG Signature</a>
+signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
+ <li>Git tag: <a href="https://github.com/apache/orc/releases/tag/rel/release-1.1.0">fc571e77</a></li>
+ <li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.sha256">8beea2be</a></li>
+</ul>
+
+<p>The major new features:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-1">ORC-1</a> Copy the Java ORC code from Hive.</li>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-10">ORC-10</a> Fix the C++ reader to correctly read
+timestamps from timezones with different daylight savings rules.</li>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-52">ORC-52</a> Add mapred and mapreduce connectors.</li>
+</ul>
+
+<p>Known issues:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-40">ORC-40</a> Predicate push down is not implemented in C++.</li>
+</ul>
+
+ </div>
+</article>
+
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+ <ul>
+ <li class="">
+ <a href="/news/">All News</a>
+ </li>
+ <li class="">
+ <a href="/news/releases/">ORC Releases</a>
+ </li>
+ </ul>
+ <h4>Recent Releases</h4>
+ <ul>
+
+ <li class="current">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
+ <a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
+ </li>
+
+ </ul>
+ <h4>Other News</h4>
+ <ul>
+
+
+
+
+
+
+ <li class="">
+ <a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
+ </li>
+
+
+
+ <li class="">
+ <a href="/news/2015/06/26/new-logo/">ORC adopts new logo</a>
+ </li>
+
+
+
+ <li class="">
+ <a href="/news/2015/05/11/new-committers/">ORC adds 7 committers</a>
+ </li>
+
+
+
+ <li class="">
+ <a href="/news/2015/04/22/orc-as-tlp/">ORC becomes an Apache Top Level Project</a>
+ </li>
+
+
+ </ul>
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/index.html
----------------------------------------------------------------------
diff --git a/news/index.html b/news/index.html
index e320303..8b10962 100644
--- a/news/index.html
+++ b/news/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -106,6 +108,62 @@
<article>
<h2>
+ <a href="/news/2016/06/10/ORC-1.1.0/">
+ ORC 1.1.0 Released
+ </a>
+ </h2>
+ <span class="post-category">
+ <span class="label">
+ release
+ </span>
+ </span>
+ <div class="post-meta">
+ <span class="post-date">
+ 10 Jun 2016
+ </span>
+ <a href="https://people.apache.org/~omalley" class="post-author">
+ <img src="https://people.apache.org/~omalley//omalley.png"
+ class="avatar" alt="omalley avatar"
+ width="24" height="24">
+ omalley
+ </a>
+ </div>
+ <div class="post-content">
+ <p>The ORC team is excited to announce the release of ORC v1.1.0. This release
+contains the Java reader and writer and the native C++ ORC reader and tools.</p>
+
+<p>Release Artifacts:</p>
+
+<ul>
+ <li>Released: 10 Jun 2016</li>
+ <li>Source code: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.1.0/orc-1.1.0.tgz">orc-1.1.0.tgz</a></li>
+ <li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.asc">GPG Signature</a>
+signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
+ <li>Git tag: <a href="https://github.com/apache/orc/releases/tag/rel/release-1.1.0">fc571e77</a></li>
+ <li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.sha256">8beea2be</a></li>
+</ul>
+
+<p>The major new features:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-1">ORC-1</a> Copy the Java ORC code from Hive.</li>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-10">ORC-10</a> Fix the C++ reader to correctly read
+timestamps from timezones with different daylight savings rules.</li>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-52">ORC-52</a> Add mapred and mapreduce connectors.</li>
+</ul>
+
+<p>Known issues:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-40">ORC-40</a> Predicate push down is not implemented in C++.</li>
+</ul>
+
+ </div>
+</article>
+
+
+ <article>
+ <h2>
<a href="/news/2016/01/25/ORC-1.0.0/">
ORC 1.0.0 Released
</a>
@@ -133,9 +191,9 @@ contains the native C++ ORC reader and some tools.</p>
<p>Release Artifacts:</p>
<ul>
- <li>Source: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz">orc-1.0.0.tgz</a></li>
+ <li>Source: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.0.0/orc-1.0.0.tgz">orc-1.0.0.tgz</a></li>
<li>Git Tag:
-<a href="https://github.com/apache/orc/releases/tag/rel%2Frelease-1.0.0">rel/release-1.0.0 (704d97e)</a></li>
+<a href="https://github.com/apache/orc/releases/tag/rel/release-1.0.0">rel/release-1.0.0 (704d97e)</a></li>
<li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz.asc">GPG Signature</a>
signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
<li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz.mds">8AD5111F 0CA3B72F</a></li>
@@ -373,6 +431,10 @@ of the website.</p>
<ul>
<li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -383,6 +445,8 @@ of the website.</p>
+
+
<li class="">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/releases/index.html
----------------------------------------------------------------------
diff --git a/news/releases/index.html b/news/releases/index.html
index 1df6b56..091efc1 100644
--- a/news/releases/index.html
+++ b/news/releases/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -106,6 +108,62 @@
<article>
<h2>
+ <a href="/news/2016/06/10/ORC-1.1.0/">
+ ORC 1.1.0 Released
+ </a>
+ </h2>
+ <span class="post-category">
+ <span class="label">
+ release
+ </span>
+ </span>
+ <div class="post-meta">
+ <span class="post-date">
+ 10 Jun 2016
+ </span>
+ <a href="https://people.apache.org/~omalley" class="post-author">
+ <img src="https://people.apache.org/~omalley//omalley.png"
+ class="avatar" alt="omalley avatar"
+ width="24" height="24">
+ omalley
+ </a>
+ </div>
+ <div class="post-content">
+ <p>The ORC team is excited to announce the release of ORC v1.1.0. This release
+contains the Java reader and writer and the native C++ ORC reader and tools.</p>
+
+<p>Release Artifacts:</p>
+
+<ul>
+ <li>Released: 10 Jun 2016</li>
+ <li>Source code: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.1.0/orc-1.1.0.tgz">orc-1.1.0.tgz</a></li>
+ <li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.asc">GPG Signature</a>
+signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
+ <li>Git tag: <a href="https://github.com/apache/orc/releases/tag/rel/release-1.1.0">fc571e77</a></li>
+ <li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.sha256">8beea2be</a></li>
+</ul>
+
+<p>The major new features:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-1">ORC-1</a> Copy the Java ORC code from Hive.</li>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-10">ORC-10</a> Fix the C++ reader to correctly read
+timestamps from timezones with different daylight savings rules.</li>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-52">ORC-52</a> Add mapred and mapreduce connectors.</li>
+</ul>
+
+<p>Known issues:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-40">ORC-40</a> Predicate push down is not implemented in C++.</li>
+</ul>
+
+ </div>
+</article>
+
+
+ <article>
+ <h2>
<a href="/news/2016/01/25/ORC-1.0.0/">
ORC 1.0.0 Released
</a>
@@ -133,9 +191,9 @@ contains the native C++ ORC reader and some tools.</p>
<p>Release Artifacts:</p>
<ul>
- <li>Source: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz">orc-1.0.0.tgz</a></li>
+ <li>Source: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.0.0/orc-1.0.0.tgz">orc-1.0.0.tgz</a></li>
<li>Git Tag:
-<a href="https://github.com/apache/orc/releases/tag/rel%2Frelease-1.0.0">rel/release-1.0.0 (704d97e)</a></li>
+<a href="https://github.com/apache/orc/releases/tag/rel/release-1.0.0">rel/release-1.0.0 (704d97e)</a></li>
<li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz.asc">GPG Signature</a>
signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
<li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.0.0/orc-1.0.0.tgz.mds">8AD5111F 0CA3B72F</a></li>
@@ -191,6 +249,10 @@ reader.</li>
<ul>
<li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -201,6 +263,8 @@ reader.</li>
+
+
<li class="">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
[5/6] orc git commit: Pushing update for ORC-65
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/core-java.html
----------------------------------------------------------------------
diff --git a/docs/core-java.html b/docs/core-java.html
new file mode 100644
index 0000000..3458be9
--- /dev/null
+++ b/docs/core-java.html
@@ -0,0 +1,2167 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>Using Core Java</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Using Core Java</h1>
+ <p>The Core ORC API reads and writes ORC files into Hive\u2019s storage-api
+vectorized classes. Both Hive and MapReduce use the Core API to actually
+read and write the data.</p>
+
+<h2 id="vectorized-row-batch">Vectorized Row Batch</h2>
+
+<p>Data is passed to ORC as instances of <code>VectorizedRowBatch</code> that contain
+the data for 1024 rows. The focus is on speed and accessing the data
+fields directly. <code>cols</code> is an array of <code>ColumnVector</code> and <code>size</code> is the number
+of rows.</p>
+
+<pre><code class="language-java">package org.apache.hadoop.hive.ql.exec.vector;
+
+public class VectorizedRowBatch {
+ public ColumnVector[] cols;
+ public int size;
+ ...
+}
+</code></pre>
+
+<p><code>ColumnVector</code> is the parent type of the different kinds of columns
+and has some fields that are shared across all of the column types. In
+particular, the <code>noNulls</code> flag if there are no nulls in this column for
+this batch and the <code>isRepeating</code> flag for columns were the entire batch is the
+same value. For columns where <code>noNulls == false</code> the <code>isNull</code> array is true
+if that value is null.</p>
+
+<pre><code class="language-java">public abstract class ColumnVector {
+
+ // If the whole column vector has no nulls, this is true, otherwise false.
+ public boolean noNulls;
+
+ // If hasNulls is true, then this array contains true if the value is
+ // is null, otherwise false.
+ public boolean[] isNull;
+
+ /*
+ * True if same value repeats for whole column vector.
+ * If so, vector[0] holds the repeating value.
+ */
+ public boolean isRepeating;
+ ...
+}
+</code></pre>
+
+<p>The subtypes of <code>ColumnVector</code> are:</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>ORC Type</th>
+ <th>ColumnVector</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>array</td>
+ <td>ListColumnVector</td>
+ </tr>
+ <tr>
+ <td>binary</td>
+ <td>BytesColumnVector</td>
+ </tr>
+ <tr>
+ <td>bigint</td>
+ <td>LongColumnVector</td>
+ </tr>
+ <tr>
+ <td>boolean</td>
+ <td>LongColumnVector</td>
+ </tr>
+ <tr>
+ <td>char</td>
+ <td>BytesColumnVector</td>
+ </tr>
+ <tr>
+ <td>date</td>
+ <td>LongColumnVector</td>
+ </tr>
+ <tr>
+ <td>decimal</td>
+ <td>DecimalColumnVector</td>
+ </tr>
+ <tr>
+ <td>double</td>
+ <td>DoubleColumnVector</td>
+ </tr>
+ <tr>
+ <td>float</td>
+ <td>DoubleColumnVector</td>
+ </tr>
+ <tr>
+ <td>int</td>
+ <td>LongColumnVector</td>
+ </tr>
+ <tr>
+ <td>map</td>
+ <td>MapColumnVector</td>
+ </tr>
+ <tr>
+ <td>smallint</td>
+ <td>LongColumnVector</td>
+ </tr>
+ <tr>
+ <td>string</td>
+ <td>BytesColumnVector</td>
+ </tr>
+ <tr>
+ <td>struct</td>
+ <td>StructColumnVector</td>
+ </tr>
+ <tr>
+ <td>timestamp</td>
+ <td>TimestampColumnVector</td>
+ </tr>
+ <tr>
+ <td>tinyint</td>
+ <td>LongColumnVector</td>
+ </tr>
+ <tr>
+ <td>uniontype</td>
+ <td>UnionColumnVector</td>
+ </tr>
+ <tr>
+ <td>varchar</td>
+ <td>BytesColumnVector</td>
+ </tr>
+ </tbody>
+</table>
+
+<p><code>LongColumnVector</code> handles all of the integer types (boolean, bigint,
+date, int, smallint, and tinyint). The data is represented as an array of
+longs where each value is sign-extended as necessary.</p>
+
+<pre><code class="language-java">public class LongColumnVector extends ColumnVector {
+ public long[] vector;
+ ...
+}
+</code></pre>
+
+<p><code>TimestampColumnVector</code> handles timestamp values. The data is represented
+as an array of longs and an array of ints.</p>
+
+<pre><code class="language-java">public class TimestampColumnVector extends ColumnVector {
+
+ // the number of milliseconds since 1 Jan 1970 00:00 GMT
+ public long[] time;
+
+ // the number of nanoseconds within the second
+ public int[] nanos
+ ...
+}
+</code></pre>
+
+<p><code>DoubleColumnVector</code> handles all of the floating point types (double,
+and float). The data is represented as an array of doubles.</p>
+
+<pre><code class="language-java">public class DoubleColumnVector extends ColumnVector {
+ public double[] vector;
+ ...
+}
+</code></pre>
+
+<p><code>DecimalColumnVector</code> handles decimal columns. The data is represented
+as an array of HiveDecimalWritable. Note that this implementation is not
+performant and will likely be replaced.</p>
+
+<pre><code class="language-java">public class DecimalColumnVector extends ColumnVector {
+ public HiveDecimalWritable[] vector;
+ ...
+}
+</code></pre>
+
+<p><code>BytesColumnVector</code> handles all of the binary types (binary, char,
+string, and varchar). The data is represented as a byte array, offset,
+and length. The byte arrays may or may not be shared between values.</p>
+
+<pre><code class="language-java">public class BytesColumnVector extends ColumnVector {
+ public byte[][] vector;
+ public int[] start;
+ public int[] length;
+ ...
+}
+</code></pre>
+
+<p><code>StructColumnVector</code> handles the struct columns and represents the data as an
+array of <code>ColumnVector</code>. The value for row 5 consists of the fifth value from
+each of the <code>fields</code> values.</p>
+
+<pre><code class="language-java">public class StructColumnVector extends ColumnVector {
+ public ColumnVector[] fields;
+ ...
+}
+</code></pre>
+
+<p><code>UnionColumnVector</code> handles the union columns and represents the data
+as an array of integers that pick the subtype and a <code>fields</code> array one
+per a subtype. Only the value of the <code>fields</code> that corresponds to
+<code>tags[row]</code> is set.</p>
+
+<pre><code class="language-java">public class UnionColumnVector extends ColumnVector {
+ public int[] tags;
+ public ColumnVector[] fields;
+ ...
+}
+</code></pre>
+
+<p><code>ListColumnVector</code> handles the array columns and represents the data
+as two arrays of integers for the offset and lengths and a
+<code>ColumnVector</code> for the children values.</p>
+
+<pre><code class="language-java">public class ListColumnVector extends ColumnVector {
+ // for each row, the first offset of the child
+ public long[] offsets;
+ // for each row, the number of elements in the array
+ public long[] lengths;
+ // the offset in the child that should be used for new values
+ public int childCount;
+
+ // the values of the children
+ public ColumnVector child;
+ ...
+}
+</code></pre>
+
+<p><code>MapColumnVector</code> handles the map columns and represents the data
+as two arrays of integers for the offset and lengths and two
+<code>ColumnVector</code>s for the keys and values.</p>
+
+<pre><code class="language-java">public class ListColumnVector extends ColumnVector {
+ // for each row, the first offset of the child
+ public long[] offsets;
+ // for each row, the number of elements in the array
+ public long[] lengths;
+ // the offset in the child that should be used for new values
+ public int childCount;
+
+ // the values of the keys and values
+ public ColumnVector keys;
+ public ColumnVector values;
+ ...
+}
+</code></pre>
+
+<h2 id="writing-orc-files">Writing ORC Files</h2>
+
+<p>To write an ORC file, you need to define the schema and create a <code>Writer</code>
+with the desired filename. This example sets the required schema parameter,
+but there are many other options to control the ORC writer.</p>
+
+<pre><code class="language-java">TypeDescription schema = TypeDescription.fromString("struct<x:int,y:int>");
+Writer writer = OrcFile.createWriter(new Path("my-file.orc"),
+ OrcFile.writerOptions(conf)
+ .schema(schema));
+</code></pre>
+
+<p>Now you need to create a row batch, set the data, and write it to the file
+as the batch fills up. When the file is done, close the <code>Writer</code>.</p>
+
+<pre><code class="language-java">VectorizedRowBatch batch = schema.createRowBatch();
+LongColumnVector x = (LongColumnVector) batch.cols[0];
+LongColumnVector y = (LongColumnVector) batch.cols[1];
+for(int r=0; r < 10000; ++r) {
+ int row = batch.size++;
+ x.vector[row] = r;
+ y.vector[row] = r * 3;
+ // If the batch is full, write it out and start over.
+ if (batch.size == batch.getMaxSize()) {
+ writer.addRowBatch(batch);
+ batch.reset();
+ }
+}
+writer.close();
+</code></pre>
+
+<h2 id="reading-orc-files">Reading ORC Files</h2>
+
+<p>To read ORC files, create a <code>Reader</code> that contains the metadata about
+the file. There are a few options to the ORC reader, but far fewer than
+the writer and none of them are required. The reader has methods for
+getting the number of rows, schema, compression, etc. from the file.</p>
+
+<pre><code class="language-java">Reader reader = OrcFile.createReader(new Path("my-file.orc"),
+ OrcFile.readerOptions(conf));
+</code></pre>
+
+<p>To get the data, create a <code>RecordReader</code> object. By default, the
+RecordReader reads all rows and all columns, but there are options to
+control the data that is read.</p>
+
+<pre><code class="language-java">RecordReader rows = reader.rows();
+VectorizedRowBatch batch = reader.getSchema().createRowBatch();
+</code></pre>
+
+<p>With a <code>RecordReader</code> the user can ask for the next batch until there
+are no more left. The reader will stop the batch at certain boundaries, so the
+returned batch may not be full, but it will always contain some rows.</p>
+
+<pre><code class="language-java">while (rows.nextBatch(batch)) {
+ for(int r=0; r < batch.size; ++r) {
+ ... process row r from batch
+ }
+}
+rows.close();
+</code></pre>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/mapreduce.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/tools.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/encodings.html
----------------------------------------------------------------------
diff --git a/docs/encodings.html b/docs/encodings.html
index 7e66102..426b380 100644
--- a/docs/encodings.html
+++ b/docs/encodings.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
+ <optgroup label="Format Specification">
+
- <div class="unit four-fifths">
- <article>
- <h1>Column Encodings</h1>
- <h2 id="smallint-int-and-bigint-columns">SmallInt, Int, and BigInt Columns</h2>
-<p>All of the 16, 32, and 64 bit integer column types use the same set of
-potential encodings, which is basically whether they use RLE v1 or
-v2. If the PRESENT stream is not included, all of the values are
-present. For values that have false bits in the present stream, no
-values are included in the data stream.</p>
+
-<table>
- <thead>
- <tr>
- <th style="text-align: left">Encoding</th>
- <th style="text-align: left">Stream Kind</th>
- <th style="text-align: left">Optional</th>
- <th style="text-align: left">Contents</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td style="text-align: left">DIRECT</td>
- <td style="text-align: left">PRESENT</td>
- <td style="text-align: left">Yes</td>
- <td style="text-align: left">Boolean RLE</td>
- </tr>
- <tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Column Encodings</h1>
+ <h2 id="smallint-int-and-bigint-columns">SmallInt, Int, and BigInt Columns</h2>
+
+<p>All of the 16, 32, and 64 bit integer column types use the same set of
+potential encodings, which is basically whether they use RLE v1 or
+v2. If the PRESENT stream is not included, all of the values are
+present. For values that have false bits in the present stream, no
+values are included in the data stream.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th style="text-align: left">Encoding</th>
+ <th style="text-align: left">Stream Kind</th>
+ <th style="text-align: left">Optional</th>
+ <th style="text-align: left">Contents</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td style="text-align: left">DIRECT</td>
+ <td style="text-align: left">PRESENT</td>
+ <td style="text-align: left">Yes</td>
+ <td style="text-align: left">Boolean RLE</td>
+ </tr>
+ <tr>
<td style="text-align: left">�</td>
<td style="text-align: left">DATA</td>
<td style="text-align: left">No</td>
@@ -1110,105 +1642,334 @@ another child column for the value.</p>
</tbody>
</table>
-<h2 id="union-columns">Union Columns</h2>
+<h2 id="union-columns">Union Columns</h2>
+
+<p>Unions are encoded as the PRESENT stream and a tag stream that controls which
+potential variant is used. They have a child column for each variant of the
+union. Currently ORC union types are limited to 256 variants, which matches
+the Hive type model.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th style="text-align: left">Encoding</th>
+ <th style="text-align: left">Stream Kind</th>
+ <th style="text-align: left">Optional</th>
+ <th style="text-align: left">Contents</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td style="text-align: left">DIRECT</td>
+ <td style="text-align: left">PRESENT</td>
+ <td style="text-align: left">Yes</td>
+ <td style="text-align: left">Boolean RLE</td>
+ </tr>
+ <tr>
+ <td style="text-align: left">�</td>
+ <td style="text-align: left">DIRECT</td>
+ <td style="text-align: left">No</td>
+ <td style="text-align: left">Byte RLE</td>
+ </tr>
+ </tbody>
+</table>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/stripes.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/spec-index.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
-<p>Unions are encoded as the PRESENT stream and a tag stream that controls which
-potential variant is used. They have a child column for each variant of the
-union. Currently ORC union types are limited to 256 variants, which matches
-the Hive type model.</p>
+
-<table>
- <thead>
- <tr>
- <th style="text-align: left">Encoding</th>
- <th style="text-align: left">Stream Kind</th>
- <th style="text-align: left">Optional</th>
- <th style="text-align: left">Contents</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td style="text-align: left">DIRECT</td>
- <td style="text-align: left">PRESENT</td>
- <td style="text-align: left">Yes</td>
- <td style="text-align: left">Boolean RLE</td>
- </tr>
- <tr>
- <td style="text-align: left">�</td>
- <td style="text-align: left">DIRECT</td>
- <td style="text-align: left">No</td>
- <td style="text-align: left">Byte RLE</td>
- </tr>
- </tbody>
-</table>
+
+
+
-
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+</ul>
+
+ <h4>Installing</h4>
+
+<ul>
-
+
+
-
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/stripes.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/spec-index.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -1233,7 +1994,11 @@ the Hive type model.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -1259,6 +2024,38 @@ the Hive type model.</p>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1269,7 +2066,7 @@ the Hive type model.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -1295,10 +2092,28 @@ the Hive type model.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -1307,14 +2122,22 @@ the Hive type model.</p>
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -1337,16 +2160,12 @@ the Hive type model.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -1357,7 +2176,17 @@ the Hive type model.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -1397,6 +2226,18 @@ the Hive type model.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -1415,6 +2256,12 @@ the Hive type model.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -1429,6 +2276,10 @@ the Hive type model.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -1457,6 +2308,18 @@ the Hive type model.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -1491,6 +2354,18 @@ the Hive type model.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -1507,6 +2382,12 @@ the Hive type model.</p>
+
+
+
+
+
+
<li class="current"><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1537,6 +2418,18 @@ the Hive type model.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/file-tail.html
----------------------------------------------------------------------
diff --git a/docs/file-tail.html b/docs/file-tail.html
index 8a2e761..7caa49e 100644
--- a/docs/file-tail.html
+++ b/docs/file-tail.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>File Tail</h1>
- <p>Since HDFS does not support changing the data in a file after it is
-written, ORC stores the top level index at the end of the file. The
-overall structure of the file is given in the figure above. The
-file\u2019s tail consists of 3 parts; the file metadata, file footer and
-postscript.</p>
+ <optgroup label="Format Specification">
+
-<p>The metadata for ORC is stored using
-<a href="http://s.apache.org/protobuf_encoding">Protocol Buffers</a>, which provides
-the ability to add new fields without breaking readers. This document
-incorporates the Protobuf definition from the
-<a href="http://s.apache.org/orc_proto">ORC source code</a> and the
-reader is encouraged to review the Protobuf encoding if they need to
-understand the byte-level encoding</p>
-<h1 id="postscript">Postscript</h1>
+
-<p>The Postscript section provides the necessary information to interpret
-the rest of the file including the length of the file\u2019s Footer and
-Metadata sections, the version of the file, and the kind of general
-compression used (eg. none, zlib, or snappy). The Postscript is never
-compressed and ends one byte before the end of the file. The version
-stored in the Postscript is the lowest version of Hive that is
-guaranteed to be able to read the file and it stored as a sequence of
-the major and minor version. There are currently two versions that are
-used: [0,11] for Hive 0.11, and [0,12] for Hive 0.12 or later.</p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>File Tail</h1>
+ <p>Since HDFS does not support changing the data in a file after it is
+written, ORC stores the top level index at the end of the file. The
+overall structure of the file is given in the figure above. The
+file\u2019s tail consists of 3 parts; the file metadata, file footer and
+postscript.</p>
+
+<p>The metadata for ORC is stored using
+<a href="http://s.apache.org/protobuf_encoding">Protocol Buffers</a>, which provides
+the ability to add new fields without breaking readers. This document
+incorporates the Protobuf definition from the
+<a href="http://s.apache.org/orc_proto">ORC source code</a> and the
+reader is encouraged to review the Protobuf encoding if they need to
+understand the byte-level encoding</p>
+
+<h1 id="postscript">Postscript</h1>
+
+<p>The Postscript section provides the necessary information to interpret
+the rest of the file including the length of the file\u2019s Footer and
+Metadata sections, the version of the file, and the kind of general
+compression used (eg. none, zlib, or snappy). The Postscript is never
+compressed and ends one byte before the end of the file. The version
+stored in the Postscript is the lowest version of Hive that is
+guaranteed to be able to read the file and it stored as a sequence of
+the major and minor version. There are currently two versions that are
+used: [0,11] for Hive 0.11, and [0,12] for Hive 0.12 or later.</p>
<p>The process of reading an ORC file works backwards through the
file. Rather than making multiple short reads, the ORC reader reads
@@ -805,100 +1337,329 @@ milliseconds since the epoch (1/1/2015).</p>
<p>Binary columns store the aggregate number of bytes across all of the values.</p>
-<p><code>message BinaryStatistics {
- // sum will store the total binary blob length
- optional sint64 sum = 1;
-}
-</code></p>
+<p><code>message BinaryStatistics {
+ // sum will store the total binary blob length
+ optional sint64 sum = 1;
+}
+</code></p>
+
+<h2 id="user-metadata">User Metadata</h2>
+
+<p>The user can add arbitrary key/value pairs to an ORC file as it is
+written. The contents of the keys and values are completely
+application defined, but the key is a string and the value is
+binary. Care should be taken by applications to make sure that their
+keys are unique and in general should be prefixed with an organization
+code.</p>
+
+<p><code>message UserMetadataItem {
+ // the user defined key
+ required string name = 1;
+ // the user defined binary value
+ required bytes value = 2;
+}
+</code></p>
+
+<h2 id="file-metadata">File Metadata</h2>
+
+<p>The file Metadata section contains column statistics at the stripe
+level granularity. These statistics enable input split elimination
+based on the predicate push-down evaluated per a stripe.</p>
+
+<p><code>message StripeStatistics {
+ repeated ColumnStatistics colStats = 1;
+}
+</code></p>
+
+<p><code>message Metadata {
+ repeated StripeStatistics stripeStats = 1;
+}
+</code></p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/spec-intro.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/compression.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
-<h2 id="user-metadata">User Metadata</h2>
+
-<p>The user can add arbitrary key/value pairs to an ORC file as it is
-written. The contents of the keys and values are completely
-application defined, but the key is a string and the value is
-binary. Care should be taken by applications to make sure that their
-keys are unique and in general should be prefixed with an organization
-code.</p>
+
+
+
-<p><code>message UserMetadataItem {
- // the user defined key
- required string name = 1;
- // the user defined binary value
- required bytes value = 2;
-}
-</code></p>
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
-<h2 id="file-metadata">File Metadata</h2>
-<p>The file Metadata section contains column statistics at the stripe
-level granularity. These statistics enable input split elimination
-based on the predicate push-down evaluated per a stripe.</p>
+</ul>
-<p><code>message StripeStatistics {
- repeated ColumnStatistics colStats = 1;
-}
-</code></p>
+
+ <h4>Installing</h4>
+
-<p><code>message Metadata {
- repeated StripeStatistics stripeStats = 1;
-}
-</code></p>
+<ul>
-
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/spec-intro.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/compression.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -923,7 +1684,11 @@ based on the predicate push-down evaluated per a stripe.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -949,6 +1714,38 @@ based on the predicate push-down evaluated per a stripe.</p>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -959,7 +1756,7 @@ based on the predicate push-down evaluated per a stripe.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -985,10 +1782,28 @@ based on the predicate push-down evaluated per a stripe.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -997,14 +1812,22 @@ based on the predicate push-down evaluated per a stripe.</p>
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -1027,16 +1850,12 @@ based on the predicate push-down evaluated per a stripe.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -1047,7 +1866,17 @@ based on the predicate push-down evaluated per a stripe.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -1087,6 +1916,18 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -1105,6 +1946,12 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
+
+
<li class="current"><a href="/docs/file-tail.html">File Tail</a></li>
@@ -1119,6 +1966,10 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -1147,6 +1998,18 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -1181,6 +2044,18 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -1197,6 +2072,12 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1227,6 +2108,18 @@ based on the predicate push-down evaluated per a stripe.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/hive-config.html
----------------------------------------------------------------------
diff --git a/docs/hive-config.html b/docs/hive-config.html
index 4179348..7f4dd83 100644
--- a/docs/hive-config.html
+++ b/docs/hive-config.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
+ <optgroup label="Format Specification">
+
- <div class="unit four-fifths">
- <article>
- <h1>Hive Configuration</h1>
- <h2 id="table-properties">Table properties</h2>
-<p>Tables stored as ORC files use table properties to control their behavior. By
-using table properties, the table owner ensures that all clients store data
-with the same options.</p>
+
-<table>
- <thead>
- <tr>
- <th style="text-align: left">Key</th>
- <th style="text-align: left">Default</th>
- <th style="text-align: left">Notes</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td style="text-align: left">orc.compress</td>
- <td style="text-align: left">ZLIB</td>
- <td style="text-align: left">high level compression = {NONE, ZLIB, SNAPPY}</td>
- </tr>
- <tr>
- <td style="text-align: left">orc.compress.size</td>
- <td style="text-align: left">262,144</td>
- <td style="text-align: left">compression chunk size</td>
- </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Hive Configuration</h1>
+ <h2 id="table-properties">Table properties</h2>
+
+<p>Tables stored as ORC files use table properties to control their behavior. By
+using table properties, the table owner ensures that all clients store data
+with the same options.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th style="text-align: left">Key</th>
+ <th style="text-align: left">Default</th>
+ <th style="text-align: left">Notes</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td style="text-align: left">orc.compress</td>
+ <td style="text-align: left">ZLIB</td>
+ <td style="text-align: left">high level compression = {NONE, ZLIB, SNAPPY}</td>
+ </tr>
+ <tr>
+ <td style="text-align: left">orc.compress.size</td>
+ <td style="text-align: left">262,144</td>
+ <td style="text-align: left">compression chunk size</td>
+ </tr>
<tr>
<td style="text-align: left">orc.stripe.size</td>
<td style="text-align: left">268,435,456</td>
@@ -742,57 +1274,274 @@ with the same options.</p>
SPEED or COMPRESSION.</td>
</tr>
-</td></tr></table>
+</td></tr></table>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/hive-ddl.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/mapred.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/hive-ddl.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/spec-intro.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -817,7 +1566,11 @@ with the same options.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -843,6 +1596,38 @@ with the same options.</p>
+ <li class="current"><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -853,7 +1638,7 @@ with the same options.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -879,10 +1664,28 @@ with the same options.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -891,14 +1694,22 @@ with the same options.</p>
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -921,16 +1732,12 @@ with the same options.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -941,7 +1748,17 @@ with the same options.</p>
- <li class="current"><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -981,6 +1798,18 @@ with the same options.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -999,6 +1828,12 @@ with the same options.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -1013,6 +1848,10 @@ with the same options.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -1041,6 +1880,18 @@ with the same options.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -1075,6 +1926,18 @@ with the same options.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -1091,6 +1954,12 @@ with the same options.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1121,6 +1990,18 @@ with the same options.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
[4/6] orc git commit: Pushing update for ORC-65
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/hive-ddl.html
----------------------------------------------------------------------
diff --git a/docs/hive-ddl.html b/docs/hive-ddl.html
index 1d22e26..02a58c8 100644
--- a/docs/hive-ddl.html
+++ b/docs/hive-ddl.html
@@ -105,25 +105,19 @@
- <option value="/docs/index.html">Background</option>
-
-
+ <option value="/docs/index.html">Background</option>
-
-
-
-
@@ -140,21 +134,23 @@
-
+
-
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
- <option value="/docs/types.html">Types</option>
+
-
+
-
+
@@ -171,7 +167,7 @@
- <option value="/docs/indexes.html">Indexes</option>
+
@@ -189,8 +185,6 @@
- <option value="/docs/acid.html">ACID support</option>
-
@@ -216,16 +210,7 @@
-
-
- </optgroup>
- <optgroup label="Hive Usage">
-
-
-
-
-
@@ -238,12 +223,12 @@
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+ <option value="/docs/types.html">Types</option>
-
+
-
+
@@ -255,9 +240,7 @@
-
-
-
+
@@ -268,7 +251,7 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+ <option value="/docs/indexes.html">Indexes</option>
@@ -287,19 +270,14 @@
-
-
- </optgroup>
- <optgroup label="Format Specification">
-
-
+
-
+ <option value="/docs/acid.html">ACID support</option>
@@ -319,17 +297,11 @@
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
@@ -337,8 +309,6 @@
- <option value="/docs/file-tail.html">File Tail</option>
-
@@ -348,24 +318,35 @@
+
+
+ </optgroup>
+ <optgroup label="Installing">
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
-
+
-
+
- <option value="/docs/compression.html">Compression</option>
+
@@ -411,8 +392,6 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
@@ -422,12 +401,8 @@
-
-
-
-
-
+ <option value="/docs/releases.html">Releases</option>
@@ -442,28 +417,37 @@
+
+
+ </optgroup>
+ <optgroup label="Using in Hive">
+
+
+
-
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
@@ -505,12 +489,10 @@
-
+ <option value="/docs/hive-config.html">Hive Configuration</option>
- <option value="/docs/spec-index.html">Indexes</option>
-
@@ -518,106 +500,35 @@
-
-
- </optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Hive DDL</h1>
- <p>ORC is well integrated into Hive, so storing your istari table as ORC
-is done by adding \u201cSTORED AS ORC\u201d.</p>
-
-<p><code>CREATE TABLE istari (
- name STRING,
- color STRING
-) STORED AS ORC;
-</code></p>
-
-<p>To modify a table so that new partitions of the istari table are
-stored as ORC files:</p>
-
-<p><code>ALTER TABLE istari SET FILEFORMAT ORC;
-</code></p>
-
-<p>As of Hive 0.14, users can request an efficient merge of small ORC files
-together by issuing a CONCATENATE command on their table or partition. The
-files will be merged at the stripe level without reserializatoin.</p>
-
-<p><code>ALTER TABLE istari [PARTITION partition_spec] CONCATENATE;
-</code></p>
-
-<p>To get information about an ORC file, use the orcfiledump command.</p>
-
-<p><code>% hive --orcfiledump <path_to_file>
-</code></p>
-
-<p>As of Hive 1.1, to display the data in the ORC file, use:</p>
-
-<p><code>% hive --orcfiledump -d <path_to_file>
-</code></p>
-
-
-
-
-
-
-
+
-
-
-
-
-
+
-
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/acid.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/hive-config.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
+ </optgroup>
+ <optgroup label="Using in MapReduce">
+
-<ul>
-
-
@@ -631,23 +542,15 @@ files will be merged at the stripe level without reserializatoin.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
+ <option value="/docs/mapred.html">Using in MapRed</option>
@@ -666,18 +569,12 @@ files will be merged at the stripe level without reserializatoin.</p>
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
@@ -693,36 +590,20 @@ files will be merged at the stripe level without reserializatoin.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
+
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
+
- <h4>Hive Usage</h4>
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
-
-<ul>
-
-
-
-
+
@@ -734,8 +615,11 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+ </optgroup>
- <li class="current"><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <optgroup label="Using ORC Core">
@@ -744,10 +628,6 @@ files will be merged at the stripe level without reserializatoin.</p>
-
-
-
-
@@ -755,24 +635,16 @@ files will be merged at the stripe level without reserializatoin.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-</ul>
-
+ <option value="/docs/core-java.html">Using Core Java</option>
- <h4>Format Specification</h4>
+
-
-<ul>
-
-
+
-
+
@@ -795,8 +667,965 @@ files will be merged at the stripe level without reserializatoin.</p>
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
-
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Hive DDL</h1>
+ <p>ORC is well integrated into Hive, so storing your istari table as ORC
+is done by adding \u201cSTORED AS ORC\u201d.</p>
+
+<p><code>CREATE TABLE istari (
+ name STRING,
+ color STRING
+) STORED AS ORC;
+</code></p>
+
+<p>To modify a table so that new partitions of the istari table are
+stored as ORC files:</p>
+
+<p><code>ALTER TABLE istari SET FILEFORMAT ORC;
+</code></p>
+
+<p>As of Hive 0.14, users can request an efficient merge of small ORC files
+together by issuing a CONCATENATE command on their table or partition. The
+files will be merged at the stripe level without reserializatoin.</p>
+
+<p><code>ALTER TABLE istari [PARTITION partition_spec] CONCATENATE;
+</code></p>
+
+<p>To get information about an ORC file, use the orcfiledump command.</p>
+
+<p><code>% hive --orcfiledump <path_to_file>
+</code></p>
+
+<p>As of Hive 1.1, to display the data in the ORC file, use:</p>
+
+<p><code>% hive --orcfiledump -d <path_to_file>
+</code></p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/releases.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/hive-config.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
@@ -813,6 +1642,12 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -827,6 +1662,10 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -855,6 +1694,18 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -889,6 +1740,18 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -905,6 +1768,12 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -935,6 +1804,18 @@ files will be merged at the stripe level without reserializatoin.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/index.html
----------------------------------------------------------------------
diff --git a/docs/index.html b/docs/index.html
index f2b43fc..906c346 100644
--- a/docs/index.html
+++ b/docs/index.html
@@ -105,25 +105,19 @@
- <option value="/docs/index.html">Background</option>
-
-
+ <option value="/docs/index.html">Background</option>
-
-
-
-
@@ -140,21 +134,23 @@
-
+
-
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
- <option value="/docs/types.html">Types</option>
+
-
+
-
+
@@ -171,7 +167,7 @@
- <option value="/docs/indexes.html">Indexes</option>
+
@@ -189,8 +185,6 @@
- <option value="/docs/acid.html">ACID support</option>
-
@@ -216,16 +210,7 @@
-
-
- </optgroup>
- <optgroup label="Hive Usage">
-
-
-
-
-
@@ -238,12 +223,12 @@
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+ <option value="/docs/types.html">Types</option>
-
+
-
+
@@ -255,9 +240,7 @@
-
-
-
+
@@ -268,7 +251,7 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+ <option value="/docs/indexes.html">Indexes</option>
@@ -287,19 +270,14 @@
-
-
- </optgroup>
- <optgroup label="Format Specification">
-
-
+
-
+ <option value="/docs/acid.html">ACID support</option>
@@ -319,17 +297,11 @@
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
@@ -337,8 +309,6 @@
- <option value="/docs/file-tail.html">File Tail</option>
-
@@ -348,24 +318,35 @@
+
+
+ </optgroup>
+ <optgroup label="Installing">
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
-
+
-
+
- <option value="/docs/compression.html">Compression</option>
+
@@ -411,8 +392,6 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
@@ -422,12 +401,8 @@
-
-
-
-
-
+ <option value="/docs/releases.html">Releases</option>
@@ -442,28 +417,37 @@
+
+
+ </optgroup>
+ <optgroup label="Using in Hive">
+
+
+
-
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
@@ -505,11 +489,13 @@
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
- <option value="/docs/spec-index.html">Indexes</option>
+
@@ -518,92 +504,31 @@
-
-
- </optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Background</h1>
- <p>Back in January 2013, we created ORC files as part of the initiative
-to massively speed up Apache Hive and improve the storage efficiency
-of data stored in Apache Hadoop. The focus was on enabling high speed
-processing and reducing file sizes.</p>
-
-<p>ORC is a self-describing type-aware columnar file format designed for
-Hadoop workloads. It is optimized for large streaming reads, but with
-integrated support for finding required rows quickly. Storing data in
-a columnar format lets the reader read, decompress, and process only
-the values that are required for the current query. Because ORC files
-are type-aware, the writer chooses the most appropriate encoding for
-the type and builds an internal index as the file is written.</p>
-
-<p>Predicate pushdown uses those indexes to determine which stripes in a
-file need to be read for a particular query and the row indexes can
-narrow the search to a particular set of 10,000 rows. ORC supports the
-complete set of types in Hive, including the complex types: structs,
-lists, maps, and unions.</p>
-
-<p>Many large Hadoop users have adopted ORC. For instance, Facebook uses
-ORC to <a href="http://s.apache.org/fb-scaling-300-pb">save tens of petabytes</a>
-in their data warehouse and demonstrated that ORC is <a href="http://s.apache.org/presto-orc">significantly
-faster</a> than RC File or Parquet. Yahoo
-uses ORC to store their production data and has released some of their
-<a href="http://s.apache.org/yahoo-orc">benchmark results</a>.</p>
-
-<p>ORC files are divided in to <em>stripes</em> that are roughly 64MB by
-default. The stripes in a file are independent of each other and form
-the natural unit of distributed work. Within each stripe, the columns
-are separated from each other so the reader can read just the columns
-that are required.</p>
-
-
-
-
-
-
-
+
+
+
+
+
+
+
- <div class="section-nav">
- <div class="left align-right">
-
- <span class="prev disabled">Back</span>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/types.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
+ </optgroup>
+ <optgroup label="Using in MapReduce">
+
-<ul>
-
-
@@ -617,23 +542,15 @@ that are required.</p>
- <li class="current"><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
+ <option value="/docs/mapred.html">Using in MapRed</option>
@@ -652,18 +569,12 @@ that are required.</p>
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
@@ -679,36 +590,20 @@ that are required.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
+
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
+
- <h4>Hive Usage</h4>
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
-
-<ul>
-
-
-
-
+
@@ -720,8 +615,11 @@ that are required.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <optgroup label="Using ORC Core">
@@ -730,8 +628,6 @@ that are required.</p>
-
-
@@ -739,26 +635,18 @@ that are required.</p>
-
+ <option value="/docs/core-java.html">Using Core Java</option>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-</ul>
-
+
- <h4>Format Specification</h4>
+
-
-<ul>
-
-
+
-
+
@@ -780,8 +668,11 @@ that are required.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+ <optgroup label="Tools">
@@ -790,7 +681,7 @@ that are required.</p>
-
+
@@ -799,16 +690,955 @@ that are required.</p>
- <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Background</h1>
+ <p>Back in January 2013, we created ORC files as part of the initiative
+to massively speed up Apache Hive and improve the storage efficiency
+of data stored in Apache Hadoop. The focus was on enabling high speed
+processing and reducing file sizes.</p>
+
+<p>ORC is a self-describing type-aware columnar file format designed for
+Hadoop workloads. It is optimized for large streaming reads, but with
+integrated support for finding required rows quickly. Storing data in
+a columnar format lets the reader read, decompress, and process only
+the values that are required for the current query. Because ORC files
+are type-aware, the writer chooses the most appropriate encoding for
+the type and builds an internal index as the file is written.</p>
+
+<p>Predicate pushdown uses those indexes to determine which stripes in a
+file need to be read for a particular query and the row indexes can
+narrow the search to a particular set of 10,000 rows. ORC supports the
+complete set of types in Hive, including the complex types: structs,
+lists, maps, and unions.</p>
+
+<p>Many large Hadoop users have adopted ORC. For instance, Facebook uses
+ORC to <a href="http://s.apache.org/fb-scaling-300-pb">save tens of petabytes</a>
+in their data warehouse and demonstrated that ORC is <a href="http://s.apache.org/presto-orc">significantly
+faster</a> than RC File or Parquet. Yahoo
+uses ORC to store their production data and has released some of their
+<a href="http://s.apache.org/yahoo-orc">benchmark results</a>.</p>
+
+<p>ORC files are divided in to <em>stripes</em> that are roughly 64MB by
+default. The stripes in a file are independent of each other and form
+the natural unit of distributed work. Within each stripe, the columns
+are separated from each other so the reader can read just the columns
+that are required.</p>
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+ <span class="prev disabled">Back</span>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/adopters.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -841,6 +1671,18 @@ that are required.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -875,6 +1717,18 @@ that are required.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -891,6 +1745,12 @@ that are required.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -921,6 +1781,18 @@ that are required.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/indexes.html
----------------------------------------------------------------------
diff --git a/docs/indexes.html b/docs/indexes.html
index cfbd69d..acd040b 100644
--- a/docs/indexes.html
+++ b/docs/indexes.html
@@ -105,25 +105,19 @@
- <option value="/docs/index.html">Background</option>
-
-
+ <option value="/docs/index.html">Background</option>
-
-
-
-
@@ -140,21 +134,23 @@
-
+
-
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
- <option value="/docs/types.html">Types</option>
+
-
+
-
+
@@ -171,7 +167,7 @@
- <option value="/docs/indexes.html">Indexes</option>
+
@@ -189,8 +185,6 @@
- <option value="/docs/acid.html">ACID support</option>
-
@@ -216,16 +210,7 @@
-
-
- </optgroup>
- <optgroup label="Hive Usage">
-
-
-
-
-
@@ -238,12 +223,12 @@
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+ <option value="/docs/types.html">Types</option>
-
+
-
+
@@ -255,9 +240,7 @@
-
-
-
+
@@ -268,7 +251,7 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+ <option value="/docs/indexes.html">Indexes</option>
@@ -287,19 +270,14 @@
-
-
- </optgroup>
- <optgroup label="Format Specification">
-
-
+
-
+ <option value="/docs/acid.html">ACID support</option>
@@ -319,17 +297,11 @@
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
@@ -337,8 +309,6 @@
- <option value="/docs/file-tail.html">File Tail</option>
-
@@ -348,24 +318,35 @@
+
+
+ </optgroup>
+ <optgroup label="Installing">
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
-
+
-
+
- <option value="/docs/compression.html">Compression</option>
+
@@ -411,8 +392,6 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
@@ -422,12 +401,8 @@
-
-
-
-
-
+ <option value="/docs/releases.html">Releases</option>
@@ -442,28 +417,37 @@
+
+
+ </optgroup>
+ <optgroup label="Using in Hive">
+
+
+
-
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
@@ -505,11 +489,11 @@
-
+ <option value="/docs/hive-config.html">Hive Configuration</option>
- <option value="/docs/spec-index.html">Indexes</option>
+
@@ -518,97 +502,33 @@
-
-
- </optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Indexes</h1>
- <p>ORC provides three level of indexes within each file:</p>
-
-<ul>
- <li>file level - statistics about the values in each column across the entire
-file</li>
- <li>stripe level - statistics about the values in each column for each stripe</li>
- <li>row level - statistics about the values in each column for each set of
-10,000 rows within a stripe</li>
-</ul>
-
-<p>The file and stripe level column statistics are in the file footer so
-that they are easy to access to determine if the rest of the file
-needs to be read at all. Row level indexes include both the column
-statistics for each row group and the position for seeking to the
-start of the row group.</p>
-
-<p>Column statistics always contain the count of values and whether there
-are null values present. Most other primitive types include the
-minimum and maximum values and for numeric types the sum. As of Hive
-1.2, the indexes can include bloom filters, which provide a much more
-selective filter.</p>
-
-<p>The indexes at all levels are used by the reader using Search
-ARGuments or SARGs, which are simplified expressions that restrict the
-rows that are of interest. For example, if a query was looking for
-people older than 100 years old, the SARG would be \u201cage > 100\u201d and
-only files, stripes, or row groups that had people over 100 years old
-would be read.</p>
-
-
-
-
-
-
-
+
-
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/types.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/acid.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
+ </optgroup>
+ <optgroup label="Using in MapReduce">
+
-<ul>
-
-
@@ -622,23 +542,15 @@ would be read.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
+ <option value="/docs/mapred.html">Using in MapRed</option>
@@ -657,18 +569,12 @@ would be read.</p>
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
@@ -684,36 +590,20 @@ would be read.</p>
- <li class="current"><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
+
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
+
- <h4>Hive Usage</h4>
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
-
-<ul>
-
-
-
-
+
@@ -725,8 +615,11 @@ would be read.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <optgroup label="Using ORC Core">
@@ -735,8 +628,6 @@ would be read.</p>
-
-
@@ -744,26 +635,18 @@ would be read.</p>
-
+ <option value="/docs/core-java.html">Using Core Java</option>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-</ul>
-
+
- <h4>Format Specification</h4>
+
-
-<ul>
-
-
+
-
+
@@ -785,8 +668,11 @@ would be read.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+ <optgroup label="Tools">
@@ -795,7 +681,7 @@ would be read.</p>
-
+
@@ -804,16 +690,963 @@ would be read.</p>
- <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Indexes</h1>
+ <p>ORC provides three level of indexes within each file:</p>
+
+<ul>
+ <li>file level - statistics about the values in each column across the entire
+file</li>
+ <li>stripe level - statistics about the values in each column for each stripe</li>
+ <li>row level - statistics about the values in each column for each set of
+10,000 rows within a stripe</li>
+</ul>
+
+<p>The file and stripe level column statistics are in the file footer so
+that they are easy to access to determine if the rest of the file
+needs to be read at all. Row level indexes include both the column
+statistics for each row group and the position for seeking to the
+start of the row group.</p>
+
+<p>Column statistics always contain the count of values and whether there
+are null values present. Most other primitive types include the
+minimum and maximum values and for numeric types the sum. As of Hive
+1.2, the indexes can include bloom filters, which provide a much more
+selective filter.</p>
+
+<p>The indexes at all levels are used by the reader using Search
+ARGuments or SARGs, which are simplified expressions that restrict the
+rows that are of interest. For example, if a query was looking for
+people older than 100 years old, the SARG would be \u201cage > 100\u201d and
+only files, stripes, or row groups that had people over 100 years old
+would be read.</p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/types.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/acid.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -846,6 +1679,18 @@ would be read.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -880,6 +1725,18 @@ would be read.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -896,6 +1753,12 @@ would be read.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -926,6 +1789,18 @@ would be read.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/mapred.html
----------------------------------------------------------------------
diff --git a/docs/mapred.html b/docs/mapred.html
new file mode 100644
index 0000000..698c2f6
--- /dev/null
+++ b/docs/mapred.html
@@ -0,0 +1,2106 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>Using in MapRed</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Using in MapRed</h1>
+ <p>This page describes how to read and write ORC files from Hadoop\u2019s
+older org.apache.hadoop.mapred MapReduce APIs. If you want to use the
+new org.apache.hadoop.mapreduce API, please look at the <a href="/docs/mapreduce.html">next
+page</a>.</p>
+
+<h2 id="reading-orc-files">Reading ORC files</h2>
+
+<p>Add ORC and your desired version of Hadoop to your <code>pom.xml</code>:</p>
+
+<pre><code class="language-xml"><dependencies>
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-mapreduce</artifactId>
+ <version>1.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>2.7.0</version>
+ </dependency>
+</dependencies>
+</code></pre>
+
+<p>Set the minimal properties in your JobConf:</p>
+
+<ul>
+ <li><strong>mapreduce.job.inputformat.class</strong> = org.apache.orc.mapred.OrcInputFormat</li>
+ <li><strong>mapreduce.input.fileinputformat.inputdir</strong> = your input directory</li>
+</ul>
+
+<p>ORC files contain a series of values of the same type and that type
+schema is encoded in the file. Because the ORC files are
+self-describing, the reader always knows how to correctly interpret
+the data. All of the ORC files written by Hive and most of the others have
+a struct as the value type.</p>
+
+<p>Your Mapper class will receive org.apache.hadoop.io.NullWritable as
+the key and a value based on the table below expanded recursively.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>ORC Type</th>
+ <th>Writable Type</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>array</td>
+ <td>org.apache.orc.mapred.OrcList</td>
+ </tr>
+ <tr>
+ <td>binary</td>
+ <td>org.apache.hadoop.io.BytesWritable</td>
+ </tr>
+ <tr>
+ <td>bigint</td>
+ <td>org.apache.hadoop.io.LongWritable</td>
+ </tr>
+ <tr>
+ <td>boolean</td>
+ <td>org.apache.hadoop.io.BooleanWritable</td>
+ </tr>
+ <tr>
+ <td>char</td>
+ <td>org.apache.hadoop.io.Text</td>
+ </tr>
+ <tr>
+ <td>date</td>
+ <td>org.apache.hadoop.hive.serde2.io.DateWritable</td>
+ </tr>
+ <tr>
+ <td>decimal</td>
+ <td>org.apache.hadoop.hive.serde2.io.HiveDecimalWritable</td>
+ </tr>
+ <tr>
+ <td>double</td>
+ <td>org.apache.hadoop.io.DoubleWritable</td>
+ </tr>
+ <tr>
+ <td>float</td>
+ <td>org.apache.hadoop.io.FloatWritable</td>
+ </tr>
+ <tr>
+ <td>int</td>
+ <td>org.apache.hadoop.io.IntWritable</td>
+ </tr>
+ <tr>
+ <td>map</td>
+ <td>org.apache.orc.mapred.OrcMap</td>
+ </tr>
+ <tr>
+ <td>smallint</td>
+ <td>org.apache.hadoop.io.ShortWritable</td>
+ </tr>
+ <tr>
+ <td>string</td>
+ <td>org.apache.hadoop.io.Text</td>
+ </tr>
+ <tr>
+ <td>struct</td>
+ <td>org.apache.orc.mapred.OrcStruct</td>
+ </tr>
+ <tr>
+ <td>timestamp</td>
+ <td>org.apache.orc.mapred.OrcTimestamp</td>
+ </tr>
+ <tr>
+ <td>tinyint</td>
+ <td>org.apache.hadoop.io.ByteWritable</td>
+ </tr>
+ <tr>
+ <td>uniontype</td>
+ <td>org.apache.orc.mapred.OrcUnion</td>
+ </tr>
+ <tr>
+ <td>varchar</td>
+ <td>org.apache.hadoop.io.Text</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Let\u2019s assume that your input directory contains ORC files with the
+schema <code>struct<s:string,i:int></code> and you want to use the string field
+as the key to the MapReduce shuffle and the integer as the value. The
+mapper code would look like:</p>
+
+<pre><code class="language-java">public class MyMapper
+ implements Mapper<NullWritable,OrcStruct,Text,IntWritable> {
+
+ // Input should be: struct<s:string,i:int>
+ public void map(NullWritable key, OrcStruct value,
+ OutputCollector<Text,IntWritable> output,
+ Reporter reporter) throws IOException {
+ output.collect((Text) value.getFieldValue(0),
+ (IntWritable) value.getFieldValue(1));
+ }
+
+ public void configure(JobConf conf) { }
+
+ public void close() { }
+}
+</code></pre>
+
+<h2 id="writing-orc-files">Writing ORC files</h2>
+
+<p>To write ORC files from your MapReduce job, you\u2019ll need to set</p>
+
+<ul>
+ <li><strong>mapreduce.job.outputformat.class</strong> = org.apache.orc.mapred.OrcOutputFormat</li>
+ <li><strong>mapreduce.output.fileoutputformat.outputdir</strong> = your output directory</li>
+ <li><strong>orc.mapred.output.schema</strong> = the schema to write to the ORC file</li>
+</ul>
+
+<p>The reducer needs to create the Writable value to be put into the ORC
+file and typically uses the OrcStruct.createValue(TypeDescription)
+function. For our example, let\u2019s assume that the shuffle types are
+(Text, IntWritable) from the previous section and the reduce should
+gather the integer for each key together and write them as a list. The
+output schema would be <code>struct<key:string,ints:array<int>></code>. As always
+with MapReduce, if your method stores the values, you need to copy their
+value before getting the next.</p>
+
+<pre><code class="language-java">public static class MyReducer
+ implements Reducer<Text,IntWritable,NullWritable,OrcStruct> {
+
+ private TypeDescription schema =
+ TypeDescription.fromString("struct<key:string,ints:array<int>>");
+ // createValue creates the correct value type for the schema
+ private OrcStruct pair = (OrcStruct) OrcStruct.createValue(schema);
+ // get a handle to the list of ints
+ private OrcList<IntWritable> values =
+ (OrcList<IntWritable>) pair.getFieldValue(1);
+ private final NullWritable nada = NullWritable.get();
+
+ public void reduce(Text key, Iterator<IntWritable> iterator,
+ OutputCollector<NullWritable, OrcStruct> output,
+ Reporter reporter) throws IOException {
+ pair.setFieldValue(0, key);
+ values.clear();
+ while (iterator.hasNext()) {
+ values.add(new IntWritable(iterator.next().get()));
+ }
+ output.collect(nada, pair);
+ }
+
+ public void configure(JobConf conf) { }
+
+ public void close() { }
+}
+</code></pre>
+
+<h2 id="sending-orcstruct-orclist-orcmap-or-orcunion-through-the-shuffle">Sending OrcStruct, OrcList, OrcMap, or OrcUnion through the Shuffle</h2>
+
+<p>In the previous examples, only the Hadoop types were sent through the
+MapReduce shuffle. The complex ORC types, since they are generic
+types, need to have their full type information provided to create the
+object. To enable MapReduce to properly instantiate the OrcStruct and
+other ORC types, we need to wrap it in either an OrcKey for the
+shuffle key or OrcValue for the shuffle value.</p>
+
+<p>To send two OrcStructs through the shuffle, define the following properties
+in the JobConf:</p>
+
+<ul>
+ <li><strong>mapreduce.map.output.key.class</strong> = org.apache.orc.mapred.OrcKey</li>
+ <li><strong>orc.mapred.map.output.key.schema</strong> = the shuffle key\u2019s schema</li>
+ <li><strong>mapreduce.map.output.value.class</strong> = org.apache.orc.mapred.OrcValue</li>
+ <li><strong>orc.mapred.map.output.value.schema</strong> = the shuffle value\u2019s schema</li>
+</ul>
+
+<p>The mapper just adds an OrcKey and OrcWrapper around the key and value
+respectively. These objects should be created once and reused as the mapper
+runs.</p>
+
+<pre><code class="language-java">public static class MyMapperWithShuffle
+ implements Mapper<NullWritable,OrcStruct,OrcKey,OrcValue> {
+
+ // create wrapper objects
+ private OrcKey keyWrapper = new OrcKey();
+ private OrcValue valueWrapper = new OrcValue();
+
+ // create a new structure to pass as the value in the shuffle
+ private OrcStruct outStruct = (OrcStruct) OrcStruct.createValue
+ (TypeDescription.fromString("struct<i:int,j:int>"));
+
+ // get the two fields of the outStruct
+ private IntWritable i = (IntWritable) outStruct.getFieldValue("i");
+ private IntWritable j = (IntWritable) outStruct.getFieldValue("j");
+
+ // Assume the input has type: struct<s:string,i:int>
+ public void map(NullWritable key, OrcStruct value,
+ OutputCollector<OrcKey,OrcValue> output,
+ Reporter reporter) throws IOException {
+ keyWrapper.key = value;
+ valueWrapper.value = outStruct;
+ int val = ((IntWritable) value.getFieldValue("i")).get();
+ i.set(val * 2);
+ j.set(val * val);
+ output.collect(keyWrapper, valueWrapper);
+ }
+
+ public void configure(JobConf conf) { }
+
+ public void close() { }
+}
+</code></pre>
+
+<p>The reducer code accesses the underlying OrcStructs by using the
+OrcKey.key and OrcValue.value fields.</p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/hive-config.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/mapreduce.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
[6/6] orc git commit: Pushing update for ORC-65
Posted by om...@apache.org.
Pushing update for ORC-65
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/1eb37b72
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/1eb37b72
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/1eb37b72
Branch: refs/heads/asf-site
Commit: 1eb37b7278350f40af35b45487aa56a0e8e4139e
Parents: 6ef32bc
Author: Owen O'Malley <om...@apache.org>
Authored: Sat Jun 11 11:14:54 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Sat Jun 11 11:14:54 2016 -0700
----------------------------------------------------------------------
README.md | 7 -
css/screen.css | 56 +-
docs/acid.html | 1101 ++++++++++--
docs/adopters.html | 1884 ++++++++++++++++++++
docs/building.html | 1915 ++++++++++++++++++++
docs/compression.html | 1235 +++++++++++--
docs/core-java.html | 2167 +++++++++++++++++++++++
docs/encodings.html | 1129 ++++++++++--
docs/file-tail.html | 1129 ++++++++++--
docs/hive-config.html | 1045 ++++++++++-
docs/hive-ddl.html | 1279 ++++++++++---
docs/index.html | 1256 +++++++++++--
docs/indexes.html | 1263 +++++++++++--
docs/mapred.html | 2106 ++++++++++++++++++++++
docs/mapreduce.html | 2090 ++++++++++++++++++++++
docs/releases.html | 1896 ++++++++++++++++++++
docs/run-length.html | 1115 ++++++++++--
docs/spec-index.html | 1067 ++++++++++-
docs/spec-intro.html | 1253 +++++++++++--
docs/stripes.html | 1065 ++++++++++-
docs/tools.html | 2111 ++++++++++++++++++++++
docs/types.html | 1283 +++++++++++---
downloads/index.html | 144 --
img/asf_logo.png | Bin 0 -> 21243 bytes
index.html | 47 +
news/2015/04/22/orc-as-tlp/index.html | 8 +
news/2015/05/11/new-committers/index.html | 8 +
news/2015/06/26/new-logo/index.html | 8 +
news/2015/11/19/aliaksei-on-pmc/index.html | 8 +
news/2016/01/25/ORC-1.0.0/index.html | 12 +-
news/2016/06/10/ORC-1.1.0/index.html | 273 +++
news/index.html | 68 +-
news/releases/index.html | 68 +-
33 files changed, 28062 insertions(+), 2034 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/README.md
----------------------------------------------------------------------
diff --git a/README.md b/README.md
deleted file mode 100644
index 9c7e9ce..0000000
--- a/README.md
+++ /dev/null
@@ -1,7 +0,0 @@
-# Apache ORC html branch
-
-This branch contains the generated HTML for the
-[ORC website](https://orc.apache.org).
-
-You are probably looking for the
-[ORC master branch](https://github.com/apache/orc/tree/master).
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/css/screen.css
----------------------------------------------------------------------
diff --git a/css/screen.css b/css/screen.css
index 0dd81c7..e742a78 100644
--- a/css/screen.css
+++ b/css/screen.css
@@ -729,7 +729,7 @@ footer {
font-size: 3.2em; } }
/* Quickstart */
.quickstart {
- background-color: #3F1F1F;
+ background-color: #242424;
color: #fff;
margin: 60px 0;
-webkit-box-shadow: inset 0 3px 10px rgba(0, 0, 0, 0.4);
@@ -737,10 +737,10 @@ footer {
box-shadow: inset 0 3px 10px rgba(0, 0, 0, 0.4); }
.quickstart .content {
padding: 0; }
- .quickstart h3 {
- font-size: 24px;
- line-height: 24px;
- margin-top: 20px;
+ .quickstart .side {
+ font-size: 22px;
+ line-height: 22px;
+ margin-top: 10px;
text-shadow: 0 1px 3px rgba(0, 0, 0, 0.8); }
.quickstart .code {
font-size: 12px;
@@ -826,27 +826,39 @@ footer {
.quickstart .code .output {
color: #888; }
+/* Apache bar */
+.apache .pane {
+ background-color: #3e3e3e;
+ -webkit-border-radius: 10px;
+ -moz-border-radius: 10px;
+ border-radius: 10px;
+ text-shadow: none;
+ position: relative;
+ padding: 0 20px 30px; }
+.apache img {
+ margin: -30px 0 0;
+ width: 180px;
+ height: 150px; }
+.apache h2 {
+ font-size: 28px; }
+.apache p,
+.apache a {
+ font-size: 16px; }
+.apache p {
+ margin: .75em 0; }
+
@media (min-width: 768px) {
- .free-hosting img {
+ .apache img {
float: left;
- margin: -20px -30px -30px -50px;
- width: 300px;
- height: 251px; }
- .free-hosting .pane-content {
+ margin: 20px 20px -20px -20px;
+ width: 429px;
+ height: 175px; }
+ .apache .pane-content {
margin-top: 35px;
padding-right: 30px; }
- .free-hosting p,
- .free-hosting a {
- font-size: 18px; }
- .free-hosting .pane:after {
- content: " ";
- float: right;
- background: url(../img/footer-arrow.png) top left no-repeat;
- width: 73px;
- height: 186px;
- position: absolute;
- right: 0;
- bottom: -30px; } }
+ .apache p,
+ .apache a {
+ font-size: 18px; } }
/* Article - Used for both docs and news */
article {
background-color: #444;
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/acid.html
----------------------------------------------------------------------
diff --git a/docs/acid.html b/docs/acid.html
index 0aa4423..dd80414 100644
--- a/docs/acid.html
+++ b/docs/acid.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>ACID support</h1>
- <p>Historically, the only way to atomically add data to a table in Hive
-was to add a new partition. Updating or deleting data in partition
-required removing the old partition and adding it back with the new
-data and it wasn\u2019t possible to do atomically.</p>
-
-<p>However, user\u2019s data is continually changing and as Hive matured,
-users required reliability guarantees despite the churning data
-lake. Thus, we needed to implement ACID transactions that guarantee
-atomicity, consistency, isolation, and durability. Although we support
-ACID transactions, they are not designed to support OLTP requirements.
-It can support millions of rows updated per a transaction, but it can
-not support millions of transactions an hour.</p>
-
-<p>Additionally, we wanted to support streaming ingest in to Hive tables where
-streaming applications like Flume or Storm could write data into Hive and
-have transactions commit once a minute and queries would either see all of
-a transaction or none of it.</p>
+ <optgroup label="Format Specification">
+
-<p>HDFS is a write once file system and ORC is a write-once file format, so edits
-were implemented using base files and delta files where insert, update, and
-delete operations are recorded.</p>
-<p>Hive tables without ACID enabled have each partition in HDFS look like:</p>
+
-<table>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>ACID support</h1>
+ <p>Historically, the only way to atomically add data to a table in Hive
+was to add a new partition. Updating or deleting data in partition
+required removing the old partition and adding it back with the new
+data and it wasn\u2019t possible to do atomically.</p>
+
+<p>However, user\u2019s data is continually changing and as Hive matured,
+users required reliability guarantees despite the churning data
+lake. Thus, we needed to implement ACID transactions that guarantee
+atomicity, consistency, isolation, and durability. Although we support
+ACID transactions, they are not designed to support OLTP requirements.
+It can support millions of rows updated per a transaction, but it can
+not support millions of transactions an hour.</p>
+
+<p>Additionally, we wanted to support streaming ingest in to Hive tables where
+streaming applications like Flume or Storm could write data into Hive and
+have transactions commit once a minute and queries would either see all of
+a transaction or none of it.</p>
+
+<p>HDFS is a write once file system and ORC is a write-once file format, so edits
+were implemented using base files and delta files where insert, update, and
+delete operations are recorded.</p>
+
+<p>Hive tables without ACID enabled have each partition in HDFS look like:</p>
+
+<table>
<thead>
<tr>
<th style="text-align: left">Filename</th>
@@ -710,84 +1242,295 @@ descending). Only the first record with a currentTransaction that is
in the list of transactions to read is returned, which corresponds to
the last visible update to a row.</p>
-<p>To support streaming ingest, we add two additional features. ORC files
-may have additional footers written in to their body that is parsable
-as a complete ORC file that only includes the records already
-written. As the file is later extended the preliminary file footer
-becomes dead space within the file. Secondly, a side file named
-\u201c*_flush_length\u201d is a small file that contains a set of 8 byte
-values. The last complete 8 byte value is the end of the last
-preliminary footer. </p>
+<p>To support streaming ingest, we add two additional features. ORC files
+may have additional footers written in to their body that is parsable
+as a complete ORC file that only includes the records already
+written. As the file is later extended the preliminary file footer
+becomes dead space within the file. Secondly, a side file named
+\u201c*_flush_length\u201d is a small file that contains a set of 8 byte
+values. The last complete 8 byte value is the end of the last
+preliminary footer. </p>
+
+<p>Two properties are added to the metadata for ORC files to speed up the
+processing of the ACID tables. In particular, when a task is reading
+part of the base file for a bucket, it will use the first and last
+rowIds to find the corresponding spots in the delta files. The
+hive.acid.key.index lets the reader skip over stripes in the delta
+file that don\u2019t need to be read in this task.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th style="text-align: left">Key</th>
+ <th style="text-align: left">Meaning</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td style="text-align: left">hive.acid.stats</td>
+ <td style="text-align: left">Number of inserts, updates, and deletes comma separated</td>
+ </tr>
+ <tr>
+ <td style="text-align: left">hive.acid.key.index</td>
+ <td style="text-align: left">The last originalTransaction, bucket, rowId for each stripe</td>
+ </tr>
+ </tbody>
+</table>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/indexes.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/building.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
-<p>Two properties are added to the metadata for ORC files to speed up the
-processing of the ACID tables. In particular, when a task is reading
-part of the base file for a bucket, it will use the first and last
-rowIds to find the corresponding spots in the delta files. The
-hive.acid.key.index lets the reader skip over stripes in the delta
-file that don\u2019t need to be read in this task.</p>
+
-<table>
- <thead>
- <tr>
- <th style="text-align: left">Key</th>
- <th style="text-align: left">Meaning</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td style="text-align: left">hive.acid.stats</td>
- <td style="text-align: left">Number of inserts, updates, and deletes comma separated</td>
- </tr>
- <tr>
- <td style="text-align: left">hive.acid.key.index</td>
- <td style="text-align: left">The last originalTransaction, bucket, rowId for each stripe</td>
- </tr>
- </tbody>
-</table>
+
+
+
+
+
+
+ <li class="current"><a href="/docs/acid.html">ACID support</a></li>
+
-
+</ul>
+
+ <h4>Installing</h4>
+
+<ul>
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/indexes.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/hive-ddl.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -812,7 +1555,11 @@ file that don\u2019t need to be read in this task.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -838,6 +1585,38 @@ file that don\u2019t need to be read in this task.</p>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -848,7 +1627,7 @@ file that don\u2019t need to be read in this task.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -874,10 +1653,28 @@ file that don\u2019t need to be read in this task.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -886,14 +1683,22 @@ file that don\u2019t need to be read in this task.</p>
- <li class="current"><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -916,16 +1721,12 @@ file that don\u2019t need to be read in this task.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -936,7 +1737,17 @@ file that don\u2019t need to be read in this task.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -976,6 +1787,18 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -994,6 +1817,12 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -1008,6 +1837,10 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -1036,6 +1869,18 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -1070,6 +1915,18 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -1086,6 +1943,12 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1116,6 +1979,18 @@ file that don\u2019t need to be read in this task.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/adopters.html
----------------------------------------------------------------------
diff --git a/docs/adopters.html b/docs/adopters.html
new file mode 100644
index 0000000..e1afc02
--- /dev/null
+++ b/docs/adopters.html
@@ -0,0 +1,1884 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>ORC Adopters</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>ORC Adopters</h1>
+ <p>If your company or tool uses ORC, please let us know so that we can update
+this page.</p>
+
+<h3 id="apache-hadoophttpshadoopapacheorg"><a href="https://hadoop.apache.org/">Apache Hadoop</a></h3>
+
+<p>ORC files have always supporting reading and writing from Hadoop\u2019s MapReduce,
+but with the ORC 1.1.0 release it is now easier than ever without pulling in
+Hive\u2019s exec jar and all of its dependencies. OrcStruct now also implements
+WritableComparable and can be serialized through the MapReduce shuffle.</p>
+
+<h3 id="apache-hivehttpshiveapacheorg"><a href="https://hive.apache.org/">Apache Hive</a></h3>
+
+<p>Apache Hive was the original use case and home for ORC. ORC\u2019s strong
+type system, advanced compression, column projection, predicate push
+down, and vectorization support make Hive <a href="http://hortonworks.com/blog/orcfile-in-hdp-2-better-compression-better-performance/">perform
+better</a>
+than any other format for your data.</p>
+
+<h3 id="apache-nifihttpsnifiapacheorg"><a href="https://nifi.apache.org/">Apache Nifi</a></h3>
+
+<p>Apache Nifi is <a href="https://issues.apache.org/jira/browse/NIFI-1663">adding
+support</a> for writing
+ORC files.</p>
+
+<h3 id="apache-pighttpspigapacheorg"><a href="https://pig.apache.org/">Apache Pig</a></h3>
+
+<p>Apache Pig added support for reading and writing ORC files in <a href="http://hortonworks.com/blog/announcing-apache-pig-0-14-0/">Pig
+14.0</a>.</p>
+
+<h3 id="apache-sparkhttpssparkapacheorg"><a href="https://spark.apache.org/">Apache Spark</a></h3>
+
+<p>Apache Spark has <a href="http://hortonworks.com/blog/bringing-orc-support-into-apache-spark/">added
+support</a>
+for reading and writing ORC files with support for column project and
+predicate push down.</p>
+
+<h3 id="facebookhttpsfacebookcom"><a href="https://facebook.com">Facebook</a></h3>
+
+<p>With more than 300 PB of data, Facebook was an <a href="https://code.facebook.com/posts/229861827208629/scaling-the-facebook-data-warehouse-to-300-pb/">early adopter of
+ORC</a> and quickly put it into production.</p>
+
+<h3 id="prestohttpsprestodbio"><a href="https://prestodb.io/">Presto</a></h3>
+
+<p>The Presto team has done a lot of work <a href="https://code.facebook.com/posts/370832626374903/even-faster-data-at-the-speed-of-presto-orc/">integrating
+ORC</a> into their SQL engine.</p>
+
+<h3 id="verticahttpwww8hpcomusensoftware-solutionsadvanced-sql-big-data-analytics"><a href="http://www8.hp.com/us/en/software-solutions/advanced-sql-big-data-analytics/">Vertica</a></h3>
+
+<p>HPE Vertica has contributed significantly to the ORC C++ library. ORC
+is a significant part of Vertica SQL-on-Hadoop (VSQLoH) which brings
+the performance, reliability and standards compliance of the Vertica
+Analytic Database to the Hadoop ecosystem.</p>
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/index.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/types.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/building.html
----------------------------------------------------------------------
diff --git a/docs/building.html b/docs/building.html
new file mode 100644
index 0000000..0a8c2a8
--- /dev/null
+++ b/docs/building.html
@@ -0,0 +1,1915 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>Building ORC</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Building ORC</h1>
+ <h2 id="building-both-c-and-java">Building both C++ and Java</h2>
+
+<p>The C++ library is supported on the following operating systems:</p>
+
+<ul>
+ <li>CentOS 6 or 7</li>
+ <li>Debian 7 or 8</li>
+ <li>MacOS 10.10 or 10.11</li>
+ <li>Ubuntu 12.04, 14.04, or 16.04</li>
+</ul>
+
+<p>You\u2019ll want to install the usual set of developer tools, but at least:</p>
+
+<ul>
+ <li>cmake</li>
+ <li>g++ or clang++</li>
+ <li>java ( >= 1.7)</li>
+ <li>make</li>
+ <li>maven ( >= 3)</li>
+</ul>
+
+<p>For each version of Linux, please check the corresponding Dockerfile, which
+is in the docker subdirectory, for the list of packages required to build ORC:</p>
+
+<ul>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/centos6/Dockerfile">CentOS 6</a></li>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/centos7/Dockerfile">CentOS 7</a></li>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/debian7/Dockerfile">Debian 7</a></li>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/debian8/Dockerfile">Debian 8</a></li>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/ubuntu12/Dockerfile">Ubuntu 12</a></li>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/ubuntu14/Dockerfile">Ubuntu 14</a></li>
+ <li><a href="https://github.com/apache/orc/blob/master/docker/ubuntu16/Dockerfile">Ubuntu 16</a></li>
+</ul>
+
+<p>To build a normal release:</p>
+
+<pre><code class="language-shell">% mkdir build
+% cmake ..
+% make package test-out
+</code></pre>
+
+<p>ORC\u2019s C++ build supports three build types, which are controlled by adding
+<code>-DCMAKE_BUILD_TYPE=<type></code> to the cmake command.</p>
+
+<ul>
+ <li><strong>RELWITHDEBINFO</strong> (default) - Optimized with debug information</li>
+ <li><strong>DEBUG</strong> - Unoptimized with debug information</li>
+ <li><strong>RELEASE</strong> - Optimized with no debug information</li>
+</ul>
+
+<p>If your make command fails, it is useful to see the actual commands that make
+is invoking:</p>
+
+<pre><code class="language-shell">% make package test-out VERBOSE=1
+</code></pre>
+
+<h2 id="building-just-java">Building just Java</h2>
+
+<p>You\u2019ll need to install:</p>
+
+<ul>
+ <li>java (>= 1.7)</li>
+ <li>maven (>= 3)</li>
+</ul>
+
+<p>To build:</p>
+
+<pre><code class="language-shell">% cd java
+% mvn package
+</code></pre>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/acid.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/releases.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/compression.html
----------------------------------------------------------------------
diff --git a/docs/compression.html b/docs/compression.html
index 647baba..c30e63a 100644
--- a/docs/compression.html
+++ b/docs/compression.html
@@ -105,25 +105,19 @@
- <option value="/docs/index.html">Background</option>
-
-
+ <option value="/docs/index.html">Background</option>
-
-
-
-
@@ -140,21 +134,23 @@
-
+
-
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
- <option value="/docs/types.html">Types</option>
+
-
+
-
+
@@ -171,7 +167,7 @@
- <option value="/docs/indexes.html">Indexes</option>
+
@@ -189,8 +185,6 @@
- <option value="/docs/acid.html">ACID support</option>
-
@@ -216,16 +210,7 @@
-
-
- </optgroup>
- <optgroup label="Hive Usage">
-
-
-
-
-
@@ -238,12 +223,12 @@
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+ <option value="/docs/types.html">Types</option>
-
+
-
+
@@ -255,9 +240,7 @@
-
-
-
+
@@ -268,7 +251,7 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+ <option value="/docs/indexes.html">Indexes</option>
@@ -287,19 +270,14 @@
-
-
- </optgroup>
- <optgroup label="Format Specification">
-
-
+
-
+ <option value="/docs/acid.html">ACID support</option>
@@ -319,17 +297,11 @@
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
@@ -337,8 +309,6 @@
- <option value="/docs/file-tail.html">File Tail</option>
-
@@ -348,24 +318,35 @@
+
+
+ </optgroup>
+ <optgroup label="Installing">
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
-
+
-
+
- <option value="/docs/compression.html">Compression</option>
+
@@ -411,8 +392,6 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
@@ -422,9 +401,11 @@
-
+
+ <option value="/docs/releases.html">Releases</option>
+
-
+
@@ -436,7 +417,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using in Hive">
+
+
+
+
+
@@ -449,21 +439,23 @@
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -478,6 +470,10 @@
+
+
+
+
@@ -486,13 +482,15 @@
-
+
-
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
@@ -509,7 +507,7 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
@@ -522,110 +520,58 @@
</optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Compression</h1>
- <p>If the ORC file writer selects a generic compression codec (zlib or
-snappy), every part of the ORC file except for the Postscript is
-compressed with that codec. However, one of the requirements for ORC
-is that the reader be able to skip over compressed bytes without
-decompressing the entire stream. To manage this, ORC writes compressed
-streams in chunks with headers as in the figure below.
-To handle uncompressable data, if the compressed data is larger than
-the original, the original is stored and the isOriginal flag is
-set. Each header is 3 bytes long with (compressedLength * 2 +
-isOriginal) stored as a little endian value. For example, the header
-for a chunk that compressed to 100,000 bytes would be [0x40, 0x0d,
-0x03]. The header for 5 bytes that did not compress would be [0x0b,
-0x00, 0x00]. Each compression chunk is compressed independently so
-that as long as a decompressor starts at the top of a header, it can
-start decompressing without the previous bytes.</p>
-
-<p><img src="/img/CompressionStream.png" alt="compression streams" /></p>
-
-<p>The default compression chunk size is 256K, but writers can choose
-their own value less than 223. Larger chunks lead to better
-compression, but require more memory. The chunk size is recorded in
-the Postscript so that readers can allocate appropriately sized
-buffers.</p>
-
-<p>ORC files without generic compression write each stream directly
-with no headers.</p>
-
-
-
-
-
+ <optgroup label="Using in MapReduce">
+
-
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/file-tail.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/run-length.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
-
- </article>
- </div>
-
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+
- <h4>Overview</h4>
+
-
-<ul>
-
-
+
-
+
@@ -638,16 +584,10 @@ with no headers.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
+
@@ -658,6 +598,8 @@ with no headers.</p>
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
@@ -673,8 +615,11 @@ with no headers.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <optgroup label="Using ORC Core">
@@ -683,8 +628,6 @@ with no headers.</p>
-
-
@@ -692,7 +635,7 @@ with no headers.</p>
-
+ <option value="/docs/core-java.html">Using Core Java</option>
@@ -700,36 +643,20 @@ with no headers.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
-
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
+
- <h4>Hive Usage</h4>
+
-
-<ul>
-
-
+
-
+
@@ -741,8 +668,11 @@ with no headers.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <optgroup label="Tools">
@@ -751,7 +681,7 @@ with no headers.</p>
-
+
@@ -762,9 +692,908 @@ with no headers.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Compression</h1>
+ <p>If the ORC file writer selects a generic compression codec (zlib or
+snappy), every part of the ORC file except for the Postscript is
+compressed with that codec. However, one of the requirements for ORC
+is that the reader be able to skip over compressed bytes without
+decompressing the entire stream. To manage this, ORC writes compressed
+streams in chunks with headers as in the figure below.
+To handle uncompressable data, if the compressed data is larger than
+the original, the original is stored and the isOriginal flag is
+set. Each header is 3 bytes long with (compressedLength * 2 +
+isOriginal) stored as a little endian value. For example, the header
+for a chunk that compressed to 100,000 bytes would be [0x40, 0x0d,
+0x03]. The header for 5 bytes that did not compress would be [0x0b,
+0x00, 0x00]. Each compression chunk is compressed independently so
+that as long as a decompressor starts at the top of a header, it can
+start decompressing without the previous bytes.</p>
+
+<p><img src="/img/CompressionStream.png" alt="compression streams" /></p>
+
+<p>The default compression chunk size is 256K, but writers can choose
+their own value less than 223. Larger chunks lead to better
+compression, but require more memory. The chunk size is recorded in
+the Postscript so that readers can allocate appropriately sized
+buffers.</p>
+
+<p>ORC files without generic compression write each stream directly
+with no headers.</p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/file-tail.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/run-length.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
</ul>
@@ -802,6 +1631,18 @@ with no headers.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -820,6 +1661,12 @@ with no headers.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -834,6 +1681,10 @@ with no headers.</p>
+
+
+
+
<li class="current"><a href="/docs/compression.html">Compression</a></li>
@@ -862,6 +1713,18 @@ with no headers.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -896,6 +1759,18 @@ with no headers.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -912,6 +1787,12 @@ with no headers.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -942,6 +1823,18 @@ with no headers.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
[2/6] orc git commit: Pushing update for ORC-65
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/spec-intro.html
----------------------------------------------------------------------
diff --git a/docs/spec-intro.html b/docs/spec-intro.html
index d0637d3..b5036f1 100644
--- a/docs/spec-intro.html
+++ b/docs/spec-intro.html
@@ -105,25 +105,19 @@
- <option value="/docs/index.html">Background</option>
-
-
+ <option value="/docs/index.html">Background</option>
-
-
-
-
@@ -140,21 +134,23 @@
-
+
-
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
- <option value="/docs/types.html">Types</option>
+
-
+
-
+
@@ -171,7 +167,7 @@
- <option value="/docs/indexes.html">Indexes</option>
+
@@ -189,8 +185,6 @@
- <option value="/docs/acid.html">ACID support</option>
-
@@ -216,16 +210,7 @@
-
-
- </optgroup>
- <optgroup label="Hive Usage">
-
-
-
-
-
@@ -238,12 +223,12 @@
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+ <option value="/docs/types.html">Types</option>
-
+
-
+
@@ -255,9 +240,7 @@
-
-
-
+
@@ -268,7 +251,7 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+ <option value="/docs/indexes.html">Indexes</option>
@@ -287,19 +270,14 @@
-
-
- </optgroup>
- <optgroup label="Format Specification">
-
-
+
-
+ <option value="/docs/acid.html">ACID support</option>
@@ -319,17 +297,11 @@
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
@@ -337,8 +309,6 @@
- <option value="/docs/file-tail.html">File Tail</option>
-
@@ -348,24 +318,35 @@
+
+
+ </optgroup>
+ <optgroup label="Installing">
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
-
+
-
+
- <option value="/docs/compression.html">Compression</option>
+
@@ -411,8 +392,6 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
@@ -422,9 +401,11 @@
-
+
+ <option value="/docs/releases.html">Releases</option>
+
-
+
@@ -436,7 +417,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using in Hive">
+
+
+
+
+
@@ -449,21 +439,23 @@
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -478,6 +470,10 @@
+
+
+
+
@@ -486,13 +482,15 @@
-
+
-
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
@@ -509,7 +507,7 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
@@ -522,97 +520,41 @@
</optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Introduction</h1>
- <p>Hive\u2019s RCFile was the standard format for storing tabular data in
-Hadoop for several years. However, RCFile has limitations because it
-treats each column as a binary blob without semantics. In Hive 0.11 we
-added a new file format named Optimized Row Columnar (ORC) file that
-uses and retains the type information from the table definition. ORC
-uses type specific readers and writers that provide light weight
-compression techniques such as dictionary encoding, bit packing, delta
-encoding, and run length encoding \u2013 resulting in dramatically smaller
-files. Additionally, ORC can apply generic compression using zlib, or
-Snappy on top of the lightweight compression for even smaller
-files. However, storage savings are only part of the gain. ORC
-supports projection, which selects subsets of the columns for reading,
-so that queries reading only one column read only the required
-bytes. Furthermore, ORC files include light weight indexes that
-include the minimum and maximum values for each column in each set of
-10,000 rows and the entire file. Using pushdown filters from Hive, the
-file reader can skip entire sets of rows that aren\u2019t important for
-this query.</p>
-
-<p><img src="/img/OrcFileLayout.png" alt="ORC file structure" /></p>
-
-
-
-
-
+ <optgroup label="Using in MapReduce">
+
-
+
-
-
+
-
+
+
-
+
+
-
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/hive-config.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/file-tail.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
-
- </article>
- </div>
-
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+
- <h4>Overview</h4>
+ <option value="/docs/mapred.html">Using in MapRed</option>
-
-<ul>
-
-
-
-
-
@@ -626,15 +568,9 @@ this query.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
@@ -662,17 +598,9 @@ this query.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
-
-
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
-
-
@@ -687,8 +615,11 @@ this query.</p>
+
+
+ </optgroup>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+ <optgroup label="Using ORC Core">
@@ -697,27 +628,17 @@ this query.</p>
-
-
-
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
- <h4>Hive Usage</h4>
+
-
-<ul>
-
-
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
-
+
@@ -730,16 +651,12 @@ this query.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -750,24 +667,21 @@ this query.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
+
-</ul>
+ </optgroup>
- <h4>Format Specification</h4>
-
+ <optgroup label="Tools">
+
-<ul>
-
+
@@ -790,19 +704,952 @@ this query.</p>
- <li class="current"><a href="/docs/spec-intro.html">Introduction</a></li>
-
-
-
-
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Introduction</h1>
+ <p>Hive\u2019s RCFile was the standard format for storing tabular data in
+Hadoop for several years. However, RCFile has limitations because it
+treats each column as a binary blob without semantics. In Hive 0.11 we
+added a new file format named Optimized Row Columnar (ORC) file that
+uses and retains the type information from the table definition. ORC
+uses type specific readers and writers that provide light weight
+compression techniques such as dictionary encoding, bit packing, delta
+encoding, and run length encoding \u2013 resulting in dramatically smaller
+files. Additionally, ORC can apply generic compression using zlib, or
+Snappy on top of the lightweight compression for even smaller
+files. However, storage savings are only part of the gain. ORC
+supports projection, which selects subsets of the columns for reading,
+so that queries reading only one column read only the required
+bytes. Furthermore, ORC files include light weight indexes that
+include the minimum and maximum values for each column in each set of
+10,000 rows and the entire file. Using pushdown filters from Hive, the
+file reader can skip entire sets of rows that aren\u2019t important for
+this query.</p>
+
+<p><img src="/img/OrcFileLayout.png" alt="ORC file structure" /></p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/tools.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/file-tail.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -822,6 +1669,10 @@ this query.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -850,6 +1701,18 @@ this query.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -884,6 +1747,18 @@ this query.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -900,6 +1775,12 @@ this query.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -930,6 +1811,18 @@ this query.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/stripes.html
----------------------------------------------------------------------
diff --git a/docs/stripes.html b/docs/stripes.html
index c9d9b92..678b13f 100644
--- a/docs/stripes.html
+++ b/docs/stripes.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Stripes</h1>
- <p>The body of ORC files consists of a series of stripes. Stripes are
-large (typically ~200MB) and independent of each other and are often
-processed by different tasks. The defining characteristic for columnar
-storage formats is that the data for each column is stored separately
-and that reading data out of the file should be proportional to the
-number of columns read.</p>
-
-<p>In ORC files, each column is stored in several streams that are stored
-next to each other in the file. For example, an integer column is
-represented as two streams PRESENT, which uses one with a bit per
-value recording if the value is non-null, and DATA, which records the
-non-null values. If all of a column\u2019s values in a stripe are non-null,
-the PRESENT stream is omitted from the stripe. For binary data, ORC
-uses three streams PRESENT, DATA, and LENGTH, which stores the length
-of each value. The details of each type will be presented in the
-following subsections.</p>
+ <optgroup label="Format Specification">
+
-<h1 id="stripe-footer">Stripe Footer</h1>
-<p>The stripe footer contains the encoding of each column and the
-directory of the streams including their location.</p>
+
-<p><code>message StripeFooter {
- // the location of each stream
- repeated Stream streams = 1;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Stripes</h1>
+ <p>The body of ORC files consists of a series of stripes. Stripes are
+large (typically ~200MB) and independent of each other and are often
+processed by different tasks. The defining characteristic for columnar
+storage formats is that the data for each column is stored separately
+and that reading data out of the file should be proportional to the
+number of columns read.</p>
+
+<p>In ORC files, each column is stored in several streams that are stored
+next to each other in the file. For example, an integer column is
+represented as two streams PRESENT, which uses one with a bit per
+value recording if the value is non-null, and DATA, which records the
+non-null values. If all of a column\u2019s values in a stripe are non-null,
+the PRESENT stream is omitted from the stripe. For binary data, ORC
+uses three streams PRESENT, DATA, and LENGTH, which stores the length
+of each value. The details of each type will be presented in the
+following subsections.</p>
+
+<h1 id="stripe-footer">Stripe Footer</h1>
+
+<p>The stripe footer contains the encoding of each column and the
+directory of the streams including their location.</p>
+
+<p><code>message StripeFooter {
+ // the location of each stream
+ repeated Stream streams = 1;
// the encoding of each column
repeated ColumnEncoding columns = 2;
}
@@ -611,70 +1143,299 @@ further refined as to whether they use RLE v1 or v2.</p>
}
</code></p>
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/run-length.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/encodings.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
-
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
-
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
-
+
+
-
+
+
-
+
+
+
+
+
+
+
+
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/run-length.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/encodings.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -699,7 +1460,11 @@ further refined as to whether they use RLE v1 or v2.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -725,6 +1490,38 @@ further refined as to whether they use RLE v1 or v2.</p>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -735,7 +1532,7 @@ further refined as to whether they use RLE v1 or v2.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -761,10 +1558,28 @@ further refined as to whether they use RLE v1 or v2.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -773,14 +1588,22 @@ further refined as to whether they use RLE v1 or v2.</p>
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -803,16 +1626,12 @@ further refined as to whether they use RLE v1 or v2.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -823,7 +1642,17 @@ further refined as to whether they use RLE v1 or v2.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -863,6 +1692,18 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -881,6 +1722,12 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -895,6 +1742,10 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -923,6 +1774,18 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -957,6 +1820,18 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class="current"><a href="/docs/stripes.html">Stripes</a></li>
@@ -973,6 +1848,12 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1003,6 +1884,18 @@ further refined as to whether they use RLE v1 or v2.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/tools.html
----------------------------------------------------------------------
diff --git a/docs/tools.html b/docs/tools.html
new file mode 100644
index 0000000..fdd3c08
--- /dev/null
+++ b/docs/tools.html
@@ -0,0 +1,2111 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>Tools</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Tools</h1>
+ <h2 id="orc-contents">orc-contents</h2>
+
+<p>Displays the contents of the ORC file as a JSON document. With the
+<code>columns</code> argument only the selected columns are printed.</p>
+
+<pre><code class="language-shell">% orc-contents [--columns=1,2,...] <filename>
+</code></pre>
+
+<p>If you run it on the example file TestOrcFile.test1.orc, you\u2019ll see (without
+the line breaks within each record):</p>
+
+<pre><code class="language-shell">% orc-contents examples/TestOrcFile.test1.orc
+{"boolean1": false, "byte1": 1, "short1": 1024, "int1": 65536, \\
+ "long1": 9223372036854775807, "float1": 1, "double1": -15, \\
+ "bytes1": [0, 1, 2, 3, 4], "string1": "hi", "middle": \\
+ {"list": [{"int1": 1, "string1": "bye"}, \\
+ {"int1": 2, "string1": "sigh"}]}, \\
+ "list": [{"int1": 3, "string1": "good"}, \\
+ {"int1": 4, "string1": "bad"}], \\
+ "map": []}
+{"boolean1": true, "byte1": 100, "short1": 2048, "int1": 65536,
+ "long1": 9223372036854775807, "float1": 2, "double1": -5, \\
+ "bytes1": [], "string1": "bye", \\
+ "middle": {"list": [{"int1": 1, "string1": "bye"}, \\
+ {"int1": 2, "string1": "sigh"}]}, \\
+ "list": [{"int1": 100000000, "string1": "cat"}, \\
+ {"int1": -100000, "string1": "in"}, \\
+ {"int1": 1234, "string1": "hat"}], \\
+ "map": [{"key": "chani", "value": {"int1": 5, "string1": "chani"}}, \\
+ {"key": "mauddib", \\
+ "value": {"int1": 1, "string1": "mauddib"}}]}
+</code></pre>
+
+<h2 id="orc-metadata">orc-metadata</h2>
+
+<p>Displays the metadata of the ORC file as a JSON document. With the
+<code>verbose</code> argument additional information about the layout of the file
+is also printed.</p>
+
+<pre><code class="language-shell">% orc-metadata [-v] <filename>
+</code></pre>
+
+<p>If you run it on the example file TestOrcFile.test1.orc, you\u2019ll see:</p>
+
+<pre><code class="language-shell">% orc-metadata examples/TestOrcFile.test1.orc
+{ "name": "../examples/TestOrcFile.test1.orc",
+ "type": "struct<boolean1:boolean,byte1:tinyint,short1:smallint,
+int1:int,long1:bigint,float1:float,double1:double,bytes1:binary,
+string1:string,middle:struct<list:array<struct<int1:int,string1:
+string>>>,list:array<struct<int1:int,string1:string>>,map:map<
+string,struct<int1:int,string1:string>>>",
+ "rows": 2,
+ "stripe count": 1,
+ "format": "0.12", "writer version": "HIVE-8732",
+ "compression": "zlib", "compression block": 10000,
+ "file length": 1711,
+ "content": 1015, "stripe stats": 250, "footer": 421, "postscript": 24,
+ "row index stride": 10000,
+ "user metadata": {
+ },
+ "stripes": [
+ { "stripe": 0, "rows": 2,
+ "offset": 3, "length": 1012,
+ "index": 570, "data": 243, "footer": 199
+ }
+ ]
+}
+</code></pre>
+
+<h2 id="file-dump">file-dump</h2>
+
+<p>The org.apache.orc.tools.FileDump Java class, which is available via Hive as:</p>
+
+<pre><code class="language-shell">% hive --orcfiledump [-j] [-p] [-d] [-t] [--rowindex <cols>]
+ [--recover] [--skip-dump] [--backup-path <new path>] <file>
+</code></pre>
+
+<dl>
+ <dt>-j</dt>
+ <dd>format the output in JSON</dd>
+ <dt>-p</dt>
+ <dd>pretty print the output</dd>
+ <dt>-d</dt>
+ <dd>print the data instead of the metadata</dd>
+ <dt>-t</dt>
+ <dd>print the timezone</dd>
+ <dt>\u2013rowindex</dt>
+ <dd>print the row indexes for the comma separated list of column ids</dd>
+ <dt>\u2013recover</dt>
+ <dd>skip over corrupted values in the ORC file</dd>
+ <dt>\u2013skip-dump</dt>
+ <dd>skip dumping the metadata</dd>
+ <dt>\u2013backup-path</dt>
+ <dd>a new path to write the recovered file to</dd>
+</dl>
+
+<p>An example of the output is given below:</p>
+
+<pre><code class="language-shell">% hive --orcfiledump examples/TestOrcFile.test1.orc
+Processing data file examples/TestOrcFile.test1.orc [length: 1711]
+Structure for examples/TestOrcFile.test1.orc
+File Version: 0.12 with HIVE_8732
+Rows: 2
+Compression: ZLIB
+Compression size: 10000
+Type: struct<boolean1:boolean,byte1:tinyint,short1:smallint,int1:int,
+long1:bigint,float1:float,double1:double,bytes1:binary,string1:string,
+middle:struct<list:array<struct<int1:int,string1:string>>>,list:array<
+struct<int1:int,string1:string>>,map:map<string,struct<int1:int,string1:
+string>>>
+
+Stripe Statistics:
+ Stripe 1:
+ Column 0: count: 2 hasNull: false
+ Column 1: count: 2 hasNull: false true: 1
+ Column 2: count: 2 hasNull: false min: 1 max: 100 sum: 101
+ Column 3: count: 2 hasNull: false min: 1024 max: 2048 sum: 3072
+ Column 4: count: 2 hasNull: false min: 65536 max: 65536 sum: 131072
+ Column 5: count: 2 hasNull: false min: 9223372036854775807 max: 9223372036854775807
+ Column 6: count: 2 hasNull: false min: 1.0 max: 2.0 sum: 3.0
+ Column 7: count: 2 hasNull: false min: -15.0 max: -5.0 sum: -20.0
+ Column 8: count: 2 hasNull: false sum: 5
+ Column 9: count: 2 hasNull: false min: bye max: hi sum: 5
+ Column 10: count: 2 hasNull: false
+ Column 11: count: 2 hasNull: false
+ Column 12: count: 4 hasNull: false
+ Column 13: count: 4 hasNull: false min: 1 max: 2 sum: 6
+ Column 14: count: 4 hasNull: false min: bye max: sigh sum: 14
+ Column 15: count: 2 hasNull: false
+ Column 16: count: 5 hasNull: false
+ Column 17: count: 5 hasNull: false min: -100000 max: 100000000 sum: 99901241
+ Column 18: count: 5 hasNull: false min: bad max: in sum: 15
+ Column 19: count: 2 hasNull: false
+ Column 20: count: 2 hasNull: false min: chani max: mauddib sum: 12
+ Column 21: count: 2 hasNull: false
+ Column 22: count: 2 hasNull: false min: 1 max: 5 sum: 6
+ Column 23: count: 2 hasNull: false min: chani max: mauddib sum: 12
+
+File Statistics:
+ Column 0: count: 2 hasNull: false
+ Column 1: count: 2 hasNull: false true: 1
+ Column 2: count: 2 hasNull: false min: 1 max: 100 sum: 101
+ Column 3: count: 2 hasNull: false min: 1024 max: 2048 sum: 3072
+ Column 4: count: 2 hasNull: false min: 65536 max: 65536 sum: 131072
+ Column 5: count: 2 hasNull: false min: 9223372036854775807 max: 9223372036854775807
+ Column 6: count: 2 hasNull: false min: 1.0 max: 2.0 sum: 3.0
+ Column 7: count: 2 hasNull: false min: -15.0 max: -5.0 sum: -20.0
+ Column 8: count: 2 hasNull: false sum: 5
+ Column 9: count: 2 hasNull: false min: bye max: hi sum: 5
+ Column 10: count: 2 hasNull: false
+ Column 11: count: 2 hasNull: false
+ Column 12: count: 4 hasNull: false
+ Column 13: count: 4 hasNull: false min: 1 max: 2 sum: 6
+ Column 14: count: 4 hasNull: false min: bye max: sigh sum: 14
+ Column 15: count: 2 hasNull: false
+ Column 16: count: 5 hasNull: false
+ Column 17: count: 5 hasNull: false min: -100000 max: 100000000 sum: 99901241
+ Column 18: count: 5 hasNull: false min: bad max: in sum: 15
+ Column 19: count: 2 hasNull: false
+ Column 20: count: 2 hasNull: false min: chani max: mauddib sum: 12
+ Column 21: count: 2 hasNull: false
+ Column 22: count: 2 hasNull: false min: 1 max: 5 sum: 6
+ Column 23: count: 2 hasNull: false min: chani max: mauddib sum: 12
+
+Stripes:
+ Stripe: offset: 3 data: 243 rows: 2 tail: 199 index: 570
+ Stream: column 0 section ROW_INDEX start: 3 length 11
+ Stream: column 1 section ROW_INDEX start: 14 length 22
+ Stream: column 2 section ROW_INDEX start: 36 length 26
+ Stream: column 3 section ROW_INDEX start: 62 length 27
+ Stream: column 4 section ROW_INDEX start: 89 length 30
+ Stream: column 5 section ROW_INDEX start: 119 length 28
+ Stream: column 6 section ROW_INDEX start: 147 length 34
+ Stream: column 7 section ROW_INDEX start: 181 length 34
+ Stream: column 8 section ROW_INDEX start: 215 length 21
+ Stream: column 9 section ROW_INDEX start: 236 length 30
+ Stream: column 10 section ROW_INDEX start: 266 length 11
+ Stream: column 11 section ROW_INDEX start: 277 length 16
+ Stream: column 12 section ROW_INDEX start: 293 length 11
+ Stream: column 13 section ROW_INDEX start: 304 length 24
+ Stream: column 14 section ROW_INDEX start: 328 length 31
+ Stream: column 15 section ROW_INDEX start: 359 length 16
+ Stream: column 16 section ROW_INDEX start: 375 length 11
+ Stream: column 17 section ROW_INDEX start: 386 length 32
+ Stream: column 18 section ROW_INDEX start: 418 length 30
+ Stream: column 19 section ROW_INDEX start: 448 length 16
+ Stream: column 20 section ROW_INDEX start: 464 length 37
+ Stream: column 21 section ROW_INDEX start: 501 length 11
+ Stream: column 22 section ROW_INDEX start: 512 length 24
+ Stream: column 23 section ROW_INDEX start: 536 length 37
+ Stream: column 1 section DATA start: 573 length 5
+ Stream: column 2 section DATA start: 578 length 6
+ Stream: column 3 section DATA start: 584 length 9
+ Stream: column 4 section DATA start: 593 length 11
+ Stream: column 5 section DATA start: 604 length 12
+ Stream: column 6 section DATA start: 616 length 11
+ Stream: column 7 section DATA start: 627 length 15
+ Stream: column 8 section DATA start: 642 length 8
+ Stream: column 8 section LENGTH start: 650 length 6
+ Stream: column 9 section DATA start: 656 length 8
+ Stream: column 9 section LENGTH start: 664 length 6
+ Stream: column 11 section LENGTH start: 670 length 6
+ Stream: column 13 section DATA start: 676 length 7
+ Stream: column 14 section DATA start: 683 length 6
+ Stream: column 14 section LENGTH start: 689 length 6
+ Stream: column 14 section DICTIONARY_DATA start: 695 length 10
+ Stream: column 15 section LENGTH start: 705 length 6
+ Stream: column 17 section DATA start: 711 length 25
+ Stream: column 18 section DATA start: 736 length 18
+ Stream: column 18 section LENGTH start: 754 length 8
+ Stream: column 19 section LENGTH start: 762 length 6
+ Stream: column 20 section DATA start: 768 length 15
+ Stream: column 20 section LENGTH start: 783 length 6
+ Stream: column 22 section DATA start: 789 length 6
+ Stream: column 23 section DATA start: 795 length 15
+ Stream: column 23 section LENGTH start: 810 length 6
+ Encoding column 0: DIRECT
+ Encoding column 1: DIRECT
+ Encoding column 2: DIRECT
+ Encoding column 3: DIRECT_V2
+ Encoding column 4: DIRECT_V2
+ Encoding column 5: DIRECT_V2
+ Encoding column 6: DIRECT
+ Encoding column 7: DIRECT
+ Encoding column 8: DIRECT_V2
+ Encoding column 9: DIRECT_V2
+ Encoding column 10: DIRECT
+ Encoding column 11: DIRECT_V2
+ Encoding column 12: DIRECT
+ Encoding column 13: DIRECT_V2
+ Encoding column 14: DICTIONARY_V2[2]
+ Encoding column 15: DIRECT_V2
+ Encoding column 16: DIRECT
+ Encoding column 17: DIRECT_V2
+ Encoding column 18: DIRECT_V2
+ Encoding column 19: DIRECT_V2
+ Encoding column 20: DIRECT_V2
+ Encoding column 21: DIRECT
+ Encoding column 22: DIRECT_V2
+ Encoding column 23: DIRECT_V2
+
+File length: 1711 bytes
+Padding length: 0 bytes
+Padding ratio: 0%
+______________________________________________________________________
+</code></pre>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/core-java.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/spec-intro.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/types.html
----------------------------------------------------------------------
diff --git a/docs/types.html b/docs/types.html
index b7eaab8..d0fbca2 100644
--- a/docs/types.html
+++ b/docs/types.html
@@ -105,25 +105,19 @@
- <option value="/docs/index.html">Background</option>
-
-
+ <option value="/docs/index.html">Background</option>
-
-
-
-
@@ -140,21 +134,23 @@
-
+
-
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
- <option value="/docs/types.html">Types</option>
+
-
+
-
+
@@ -171,7 +167,7 @@
- <option value="/docs/indexes.html">Indexes</option>
+
@@ -189,8 +185,6 @@
- <option value="/docs/acid.html">ACID support</option>
-
@@ -216,16 +210,7 @@
-
-
- </optgroup>
- <optgroup label="Hive Usage">
-
-
-
-
-
@@ -238,12 +223,12 @@
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+ <option value="/docs/types.html">Types</option>
-
+
-
+
@@ -255,9 +240,7 @@
-
-
-
+
@@ -268,7 +251,7 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+ <option value="/docs/indexes.html">Indexes</option>
@@ -287,19 +270,14 @@
-
-
- </optgroup>
- <optgroup label="Format Specification">
-
-
+
-
+ <option value="/docs/acid.html">ACID support</option>
@@ -319,17 +297,11 @@
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
@@ -337,8 +309,6 @@
- <option value="/docs/file-tail.html">File Tail</option>
-
@@ -348,24 +318,35 @@
+
+
+ </optgroup>
+ <optgroup label="Installing">
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
-
+
-
+
- <option value="/docs/compression.html">Compression</option>
+
@@ -411,8 +392,6 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
@@ -422,12 +401,8 @@
-
-
-
-
-
+ <option value="/docs/releases.html">Releases</option>
@@ -442,28 +417,37 @@
+
+
+ </optgroup>
+ <optgroup label="Using in Hive">
+
+
+
-
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
@@ -505,11 +489,13 @@
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
- <option value="/docs/spec-index.html">Indexes</option>
+
@@ -518,143 +504,31 @@
-
-
- </optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Types</h1>
- <p>ORC files are completely self-describing and do not depend on the Hive
-Metastore or any other external metadata. The file includes all of the
-type and encoding information for the objects stored in the file. Because the
-file is self-contained, it does not depend on the user\u2019s environment to
-correctly interpret the file\u2019s contents.</p>
-
-<p>ORC provides a rich set of scalar and compound types:</p>
-
-<ul>
- <li>Integer
- <ul>
- <li>boolean (1 bit)</li>
- <li>tinyint (8 bit)</li>
- <li>smallint (16 bit)</li>
- <li>int (32 bit)</li>
- <li>bigint (64 bit)</li>
- </ul>
- </li>
- <li>Floating point
- <ul>
- <li>float</li>
- <li>double</li>
- </ul>
- </li>
- <li>String types
- <ul>
- <li>string</li>
- <li>char</li>
- <li>varchar</li>
- </ul>
- </li>
- <li>Binary blobs
- <ul>
- <li>binary</li>
- </ul>
- </li>
- <li>Date/time
- <ul>
- <li>timestamp</li>
- <li>date</li>
- </ul>
- </li>
- <li>Compound types
- <ul>
- <li>struct</li>
- <li>list</li>
- <li>map</li>
- <li>union</li>
- </ul>
- </li>
-</ul>
-
-<p>All ORC file are logically sequences of identically typed objects. Hive
-always uses a struct with a field for each of the top-level columns as
-the root object type, but that is not required. All types in ORC can take
-null values including the compound types.</p>
-
-<p>Compound types have children columns that hold the values for their
-sub-elements. For example, a struct column has one child column for
-each field of the struct. Lists always have a single child column for
-the element values and maps always have two child columns. Union
-columns have one child column for each of the variants.</p>
-
-<p>Given the following definition of the table Foobar, the columns in the
-file would form the given tree.</p>
-
-<p><code>create table Foobar (
- myInt int,
- myMap map<string,
- struct<myString : string,
- myDouble: double>>,
- myTime timestamp
-);
-</code></p>
-
-<p><img src="/img/TreeWriters.png" alt="ORC column structure" /></p>
-
-
-
-
-
-
-
-
+
-
+
+
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/index.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/indexes.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
+ </optgroup>
+ <optgroup label="Using in MapReduce">
+
-<ul>
-
-
@@ -668,23 +542,15 @@ file would form the given tree.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
+ <option value="/docs/mapred.html">Using in MapRed</option>
@@ -703,17 +569,13 @@ file would form the given tree.</p>
-
- <li class="current"><a href="/docs/types.html">Types</a></li>
-
-
-
+
@@ -730,16 +592,729 @@ file would form the given tree.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
+
-
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Types</h1>
+ <p>ORC files are completely self-describing and do not depend on the Hive
+Metastore or any other external metadata. The file includes all of the
+type and encoding information for the objects stored in the file. Because the
+file is self-contained, it does not depend on the user\u2019s environment to
+correctly interpret the file\u2019s contents.</p>
+
+<p>ORC provides a rich set of scalar and compound types:</p>
+
+<ul>
+ <li>Integer
+ <ul>
+ <li>boolean (1 bit)</li>
+ <li>tinyint (8 bit)</li>
+ <li>smallint (16 bit)</li>
+ <li>int (32 bit)</li>
+ <li>bigint (64 bit)</li>
+ </ul>
+ </li>
+ <li>Floating point
+ <ul>
+ <li>float</li>
+ <li>double</li>
+ </ul>
+ </li>
+ <li>String types
+ <ul>
+ <li>string</li>
+ <li>char</li>
+ <li>varchar</li>
+ </ul>
+ </li>
+ <li>Binary blobs
+ <ul>
+ <li>binary</li>
+ </ul>
+ </li>
+ <li>Date/time
+ <ul>
+ <li>timestamp</li>
+ <li>date</li>
+ </ul>
+ </li>
+ <li>Compound types
+ <ul>
+ <li>struct</li>
+ <li>list</li>
+ <li>map</li>
+ <li>union</li>
+ </ul>
+ </li>
+</ul>
+
+<p>All ORC file are logically sequences of identically typed objects. Hive
+always uses a struct with a field for each of the top-level columns as
+the root object type, but that is not required. All types in ORC can take
+null values including the compound types.</p>
+
+<p>Compound types have children columns that hold the values for their
+sub-elements. For example, a struct column has one child column for
+each field of the struct. Lists always have a single child column for
+the element values and maps always have two child columns. Union
+columns have one child column for each of the variants.</p>
+
+<p>Given the following definition of the table Foobar, the columns in the
+file would form the given tree.</p>
+
+<p><code>create table Foobar (
+ myInt int,
+ myMap map<string,
+ struct<myString : string,
+ myDouble: double>>,
+ myTime timestamp
+);
+</code></p>
+
+<p><img src="/img/TreeWriters.png" alt="ORC column structure" /></p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/adopters.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/indexes.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/acid.html">ACID support</a></li>
@@ -749,7 +1324,69 @@ file would form the given tree.</p>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
<ul>
@@ -772,6 +1409,12 @@ file would form the given tree.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -792,6 +1435,12 @@ file would form the given tree.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
@@ -799,6 +1448,168 @@ file would form the given tree.</p>
</ul>
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
<h4>Format Specification</h4>
@@ -832,6 +1643,18 @@ file would form the given tree.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -850,6 +1673,12 @@ file would form the given tree.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -864,6 +1693,10 @@ file would form the given tree.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -892,6 +1725,18 @@ file would form the given tree.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -926,6 +1771,18 @@ file would form the given tree.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -942,6 +1799,12 @@ file would form the given tree.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -972,6 +1835,18 @@ file would form the given tree.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/downloads/index.html
----------------------------------------------------------------------
diff --git a/downloads/index.html b/downloads/index.html
deleted file mode 100644
index 04040a9..0000000
--- a/downloads/index.html
+++ /dev/null
@@ -1,144 +0,0 @@
-<!DOCTYPE HTML>
-<html lang="en-US">
-<head>
- <meta charset="UTF-8">
- <title>Downloads</title>
- <meta name="viewport" content="width=device-width,initial-scale=1">
- <meta name="generator" content="Jekyll v2.4.0">
- <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
- <link rel="stylesheet" href="/css/screen.css">
- <link rel="icon" type="image/x-icon" href="/favicon.ico">
- <!--[if lt IE 9]>
- <script src="/js/html5shiv.min.js"></script>
- <script src="/js/respond.min.js"></script>
- <![endif]-->
-</head>
-
-
-<body class="wrap">
- <header role="banner">
- <nav class="mobile-nav show-on-mobiles">
- <ul>
- <li class="">
- <a href="/">Home</a>
- </li>
- <li class="">
- <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
- <span class="hide-on-mobiles">umentation</span></a>
- </li>
- <li class="">
- <a href="/talks/">Talks</a>
- </li>
- <li class="">
- <a href="/news/">News</a>
- </li>
- <li class="">
- <a href="/help/">Help</a>
- </li>
- <li class="">
- <a href="/develop/">Develop</a>
- </li>
-</ul>
-
- </nav>
- <div class="grid">
- <div class="unit one-third center-on-mobiles">
- <h1>
- <a href="/">
- <span class="sr-only">Apache ORC</span>
- <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
- </a>
- </h1>
- </div>
- <nav class="main-nav unit two-thirds hide-on-mobiles">
- <ul>
- <li class="">
- <a href="/">Home</a>
- </li>
- <li class="">
- <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
- <span class="hide-on-mobiles">umentation</span></a>
- </li>
- <li class="">
- <a href="/talks/">Talks</a>
- </li>
- <li class="">
- <a href="/news/">News</a>
- </li>
- <li class="">
- <a href="/help/">Help</a>
- </li>
- <li class="">
- <a href="/develop/">Develop</a>
- </li>
-</ul>
-
- </nav>
- </div>
-</header>
-
-
- <section class="standalone">
- <div class="grid">
-
- <div class="unit whole">
- <article>
- <h1>Downloads</h1>
- <p>We haven\u2019t made any releases as a separate project yet. Please download
-the Hive 1.1 release and use the hive-exec.jar.</p>
-
- </article>
- </div>
-
- <div class="clear"></div>
-
- </div>
-</section>
-
-
- <footer role="contentinfo">
- <p>The contents of this website are © 2016
- <a href="https://www.apache.org/">Apache Software Foundation</a>
- under the terms of the <a
- href="https://www.apache.org/licenses/LICENSE-2.0.html">
- Apache License v2</a>. Apache ORC and its logo are trademarks
- of the Apache Software Foundation.</p>
-</footer>
-
- <script>
- var anchorForId = function (id) {
- var anchor = document.createElement("a");
- anchor.className = "header-link";
- anchor.href = "#" + id;
- anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
- anchor.title = "Permalink";
- return anchor;
- };
-
- var linkifyAnchors = function (level, containingElement) {
- var headers = containingElement.getElementsByTagName("h" + level);
- for (var h = 0; h < headers.length; h++) {
- var header = headers[h];
-
- if (typeof header.id !== "undefined" && header.id !== "") {
- header.appendChild(anchorForId(header.id));
- }
- }
- };
-
- document.onreadystatechange = function () {
- if (this.readyState === "complete") {
- var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
- if (!contentBlock) {
- return;
- }
- for (var level = 1; level <= 6; level++) {
- linkifyAnchors(level, contentBlock);
- }
- }
- };
-</script>
-
-
-</body>
-</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/img/asf_logo.png
----------------------------------------------------------------------
diff --git a/img/asf_logo.png b/img/asf_logo.png
new file mode 100644
index 0000000..b20bb7f
Binary files /dev/null and b/img/asf_logo.png differ
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/index.html
----------------------------------------------------------------------
diff --git a/index.html b/index.html
index 25fcf43..85b2f49 100644
--- a/index.html
+++ b/index.html
@@ -109,6 +109,53 @@
<div class="clear"></div>
</div>
</section>
+<section class="quickstart">
+ <div class="grid">
+ <div class="unit golden-small center-on-mobiles">
+ <p class = "side">Download the <a href="docs/releases.html">ORC 1.1.0</a>
+ release.</p>
+ </div>
+ <div class="unit golden-large code">
+ <p class="title">Quickstart Documentation</p>
+ <ul class="shell">
+ <li><a href="docs/hive-ddl.html">Using with Hive</a></li>
+ <li><a href="docs/mapred.html">Using with Hadoop MapRed</a></li>
+ <li><a href="docs/mapreduce.html">Using with Hadoop MapReduce</a></li>
+ <li><a href="docs/core-java.html">Using Core ORC from Java</a></li>
+ </ul>
+ </div>
+ <div class="clear"></div>
+ </div>
+</section>
+<section class="apache">
+ <div class="grid">
+ <div class="unit whole">
+ <div class="grid pane">
+ <div class="unit whole center-on-mobiles">
+ <a href="http://www.apache.org">
+ <img src="img/asf_logo.png" width="429" height="175"
+ alt="Apache Software Foundation"></a>
+ <div class="pane-content">
+ <h2 class="center-on-mobiles">ORC is an <strong>Apache</strong>
+ project.</h2>
+ <p>Apache is a non-profit organization helping open-source
+ software projects released under the
+ <a href="http://www.apache.org/licenses/">Apache License</a>
+ and managed with
+ <a href="http://www.apache.org/foundation/how-it-works.html">
+ open governance</a>. We'd like to
+ <a href="http://www.apache.org/foundation/thanks.html">thank
+ </a> the Apache sponsors and encourage others to
+ <a href="http://www.apache.org/foundation/sponsorship.html">
+ donate</a>.
+ </p>
+ </div>
+ </div>
+ <div class="clear"></div>
+ </div>
+ </div>
+ </div>
+</section>
<footer role="contentinfo">
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/2015/04/22/orc-as-tlp/index.html
----------------------------------------------------------------------
diff --git a/news/2015/04/22/orc-as-tlp/index.html b/news/2015/04/22/orc-as-tlp/index.html
index 34e268e..fe34f79 100644
--- a/news/2015/04/22/orc-as-tlp/index.html
+++ b/news/2015/04/22/orc-as-tlp/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -181,6 +183,10 @@ together with the Java implementation.</p>
<ul>
<li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -191,6 +197,8 @@ together with the Java implementation.</p>
+
+
<li class="">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/2015/05/11/new-committers/index.html
----------------------------------------------------------------------
diff --git a/news/2015/05/11/new-committers/index.html b/news/2015/05/11/new-committers/index.html
index 131bb3a..3699e25 100644
--- a/news/2015/05/11/new-committers/index.html
+++ b/news/2015/05/11/new-committers/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -157,6 +159,10 @@ for their work on ORC. Welcome all!</p>
<ul>
<li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -167,6 +173,8 @@ for their work on ORC. Welcome all!</p>
+
+
<li class="">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/2015/06/26/new-logo/index.html
----------------------------------------------------------------------
diff --git a/news/2015/06/26/new-logo/index.html b/news/2015/06/26/new-logo/index.html
index 9e35cbf..9de7f76 100644
--- a/news/2015/06/26/new-logo/index.html
+++ b/news/2015/06/26/new-logo/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -150,6 +152,10 @@
<ul>
<li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -160,6 +166,8 @@
+
+
<li class="">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/news/2015/11/19/aliaksei-on-pmc/index.html
----------------------------------------------------------------------
diff --git a/news/2015/11/19/aliaksei-on-pmc/index.html b/news/2015/11/19/aliaksei-on-pmc/index.html
index 4060654..ba79637 100644
--- a/news/2015/11/19/aliaksei-on-pmc/index.html
+++ b/news/2015/11/19/aliaksei-on-pmc/index.html
@@ -87,6 +87,8 @@
<option value="/news/">Home</option>
<optgroup label="v1.x">
+ <option value="/news/2016/06/10/ORC-1.1.0/">ORC 1.1.0 Released</option>
+
<option value="/news/2016/01/25/ORC-1.0.0/">ORC 1.0.0 Released</option>
<option value="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</option>
@@ -153,6 +155,10 @@ more.</p>
<ul>
<li class="">
+ <a href="/news/2016/06/10/ORC-1.1.0/">Version 1.1.0</a>
+ </li>
+
+ <li class="">
<a href="/news/2016/01/25/ORC-1.0.0/">Version 1.0.0</a>
</li>
@@ -163,6 +169,8 @@ more.</p>
+
+
<li class="current">
<a href="/news/2015/11/19/aliaksei-on-pmc/">ORC adds Aliaksei Sandryhaila to PMC</a>
</li>
[3/6] orc git commit: Pushing update for ORC-65
Posted by om...@apache.org.
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/mapreduce.html
----------------------------------------------------------------------
diff --git a/docs/mapreduce.html b/docs/mapreduce.html
new file mode 100644
index 0000000..fe32bc3
--- /dev/null
+++ b/docs/mapreduce.html
@@ -0,0 +1,2090 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>Using in MapReduce</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Using in MapReduce</h1>
+ <p>This page describes how to read and write ORC files from Hadoop\u2019s
+newer org.apache.hadoop.mapreduce MapReduce APIs. If you want to use the
+older org.apache.hadoop.mapred API, please look at the <a href="/docs/mapred.html">previous
+page</a>.</p>
+
+<h2 id="reading-orc-files">Reading ORC files</h2>
+
+<p>Add ORC and your desired version of Hadoop to your <code>pom.xml</code>:</p>
+
+<pre><code class="language-xml"><dependencies>
+ <dependency>
+ <groupId>org.apache.orc</groupId>
+ <artifactId>orc-mapreduce</artifactId>
+ <version>1.1.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-core</artifactId>
+ <version>2.7.0</version>
+ </dependency>
+</dependencies>
+</code></pre>
+
+<p>Set the minimal properties in your JobConf:</p>
+
+<ul>
+ <li><strong>mapreduce.job.inputformat.class</strong> = org.apache.orc.mapreduce.OrcInputFormat</li>
+ <li><strong>mapreduce.input.fileinputformat.inputdir</strong> = your input directory</li>
+</ul>
+
+<p>ORC files contain a series of values of the same type and that type
+schema is encoded in the file. Because the ORC files are
+self-describing, the reader always knows how to correctly interpret
+the data. All of the ORC files written by Hive and most of the others have
+a struct as the value type.</p>
+
+<p>Your Mapper class will receive org.apache.hadoop.io.NullWritable as
+the key and a value based on the table below expanded recursively.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th>ORC Type</th>
+ <th>Writable Type</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>array</td>
+ <td>org.apache.orc.mapred.OrcList</td>
+ </tr>
+ <tr>
+ <td>binary</td>
+ <td>org.apache.hadoop.io.BytesWritable</td>
+ </tr>
+ <tr>
+ <td>bigint</td>
+ <td>org.apache.hadoop.io.LongWritable</td>
+ </tr>
+ <tr>
+ <td>boolean</td>
+ <td>org.apache.hadoop.io.BooleanWritable</td>
+ </tr>
+ <tr>
+ <td>char</td>
+ <td>org.apache.hadoop.io.Text</td>
+ </tr>
+ <tr>
+ <td>date</td>
+ <td>org.apache.hadoop.hive.serde2.io.DateWritable</td>
+ </tr>
+ <tr>
+ <td>decimal</td>
+ <td>org.apache.hadoop.hive.serde2.io.HiveDecimalWritable</td>
+ </tr>
+ <tr>
+ <td>double</td>
+ <td>org.apache.hadoop.io.DoubleWritable</td>
+ </tr>
+ <tr>
+ <td>float</td>
+ <td>org.apache.hadoop.io.FloatWritable</td>
+ </tr>
+ <tr>
+ <td>int</td>
+ <td>org.apache.hadoop.io.IntWritable</td>
+ </tr>
+ <tr>
+ <td>map</td>
+ <td>org.apache.orc.mapred.OrcMap</td>
+ </tr>
+ <tr>
+ <td>smallint</td>
+ <td>org.apache.hadoop.io.ShortWritable</td>
+ </tr>
+ <tr>
+ <td>string</td>
+ <td>org.apache.hadoop.io.Text</td>
+ </tr>
+ <tr>
+ <td>struct</td>
+ <td>org.apache.orc.mapred.OrcStruct</td>
+ </tr>
+ <tr>
+ <td>timestamp</td>
+ <td>org.apache.orc.mapred.OrcTimestamp</td>
+ </tr>
+ <tr>
+ <td>tinyint</td>
+ <td>org.apache.hadoop.io.ByteWritable</td>
+ </tr>
+ <tr>
+ <td>uniontype</td>
+ <td>org.apache.orc.mapred.OrcUnion</td>
+ </tr>
+ <tr>
+ <td>varchar</td>
+ <td>org.apache.hadoop.io.Text</td>
+ </tr>
+ </tbody>
+</table>
+
+<p>Let\u2019s assume that your input directory contains ORC files with the
+schema <code>struct<s:string,i:int></code> and you want to use the string field
+as the key to the MapReduce shuffle and the integer as the value. The
+mapper code would look like:</p>
+
+<pre><code class="language-java">public static class MyMapper
+ extends Mapper<NullWritable,OrcStruct,Text,IntWritable> {
+
+ // Assume the ORC file has type: struct<s:string,i:int>
+ public void map(NullWritable key, OrcStruct value,
+ Context output) throws IOException, InterruptedException {
+ // take the first field as the key and the second field as the value
+ output.write((Text) value.getFieldValue(0),
+ (IntWritable) value.getFieldValue(1));
+ }
+}
+</code></pre>
+
+<h2 id="writing-orc-files">Writing ORC files</h2>
+
+<p>To write ORC files from your MapReduce job, you\u2019ll need to set</p>
+
+<ul>
+ <li><strong>mapreduce.job.outputformat.class</strong> = org.apache.orc.mapreduce.OrcOutputFormat</li>
+ <li><strong>mapreduce.output.fileoutputformat.outputdir</strong> = your output directory</li>
+ <li><strong>orc.mapred.output.schema</strong> = the schema to write to the ORC file</li>
+</ul>
+
+<p>The reducer needs to create the Writable value to be put into the ORC
+file and typically uses the OrcStruct.createValue(TypeDescription)
+function. For our example, let\u2019s assume that the shuffle types are
+(Text, IntWritable) from the previous section and the reduce should
+gather the integer for each key together and write them as a list. The
+output schema would be <code>struct<key:string,ints:array<int>></code>. As always
+with MapReduce, if your method stores the values, you need to copy their
+value before getting the next.</p>
+
+<pre><code class="language-java">public static class MyReducer
+ extends Reducer<Text,IntWritable,NullWritable,OrcStruct> {
+
+ private TypeDescription schema =
+ TypeDescription.fromString("struct<key:string,ints:array<int>>");
+ // createValue creates the correct value type for the schema
+ private OrcStruct pair = (OrcStruct) OrcStruct.createValue(schema);
+ // get a handle to the list of ints
+ private OrcList<IntWritable> valueList =
+ (OrcList<IntWritable>) pair.getFieldValue(1);
+ private final NullWritable nada = NullWritable.get();
+
+ public void reduce(Text key, Iterable<IntWritable> values,
+ Context output
+ ) throws IOException, InterruptedException {
+ pair.setFieldValue(0, key);
+ valueList.clear();
+ for(IntWritable val: values) {
+ valueList.add(new IntWritable(val.get()));
+ }
+ output.write(nada, pair);
+ }
+}
+</code></pre>
+
+<h2 id="sending-orcstruct-orclist-orcmap-or-orcunion-through-the-shuffle">Sending OrcStruct, OrcList, OrcMap, or OrcUnion through the Shuffle</h2>
+
+<p>In the previous examples, only the Hadoop types were sent through the
+MapReduce shuffle. The complex ORC types, since they are generic
+types, need to have their full type information provided to create the
+object. To enable MapReduce to properly instantiate the OrcStruct and
+other ORC types, we need to wrap it in either an OrcKey for the
+shuffle key or OrcValue for the shuffle value.</p>
+
+<p>To send two OrcStructs through the shuffle, define the following properties
+in the JobConf:</p>
+
+<ul>
+ <li><strong>mapreduce.map.output.key.class</strong> = org.apache.orc.mapred.OrcKey</li>
+ <li><strong>orc.mapred.map.output.key.schema</strong> = the shuffle key\u2019s schema</li>
+ <li><strong>mapreduce.map.output.value.class</strong> = org.apache.orc.mapred.OrcValue</li>
+ <li><strong>orc.mapred.map.output.value.schema</strong> = the shuffle value\u2019s schema</li>
+</ul>
+
+<p>The mapper just adds an OrcKey and OrcWrapper around the key and value
+respectively. These objects should be created once and reused as the mapper
+runs.</p>
+
+<pre><code class="language-java">public static class MyMapperShuffle
+ extends Mapper<NullWritable,OrcStruct,OrcKey,OrcValue> {
+ private OrcKey keyWrapper = new OrcKey();
+ private OrcValue valueWrapper = new OrcValue();
+ private OrcStruct outStruct = (OrcStruct) OrcStruct.createValue
+ (TypeDescription.fromString("struct<i:int,j:int>"));
+ private IntWritable i = (IntWritable) outStruct.getFieldValue("i");
+ private IntWritable j = (IntWritable) outStruct.getFieldValue("j");
+
+ // Assume the input has type: struct<s:string,i:int>
+ public void map(NullWritable key, OrcStruct value,
+ Context output) throws IOException, InterruptedException {
+ keyWrapper.key = value;
+ valueWrapper.value = outStruct;
+ int val = ((IntWritable) value.getFieldValue("i")).get();
+ i.set(val * 2);
+ j.set(val * val);
+ output.write(keyWrapper, valueWrapper);
+ }
+}
+</code></pre>
+
+<p>The reducer code accesses the underlying OrcStructs by using the
+OrcKey.key and OrcValue.value fields.</p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/mapred.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/core-java.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/releases.html
----------------------------------------------------------------------
diff --git a/docs/releases.html b/docs/releases.html
new file mode 100644
index 0000000..05f8df9
--- /dev/null
+++ b/docs/releases.html
@@ -0,0 +1,1896 @@
+<!DOCTYPE HTML>
+<html lang="en-US">
+<head>
+ <meta charset="UTF-8">
+ <title>Releases</title>
+ <meta name="viewport" content="width=device-width,initial-scale=1">
+ <meta name="generator" content="Jekyll v2.4.0">
+ <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
+ <link rel="stylesheet" href="/css/screen.css">
+ <link rel="icon" type="image/x-icon" href="/favicon.ico">
+ <!--[if lt IE 9]>
+ <script src="/js/html5shiv.min.js"></script>
+ <script src="/js/respond.min.js"></script>
+ <![endif]-->
+</head>
+
+
+<body class="wrap">
+ <header role="banner">
+ <nav class="mobile-nav show-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ <div class="grid">
+ <div class="unit one-third center-on-mobiles">
+ <h1>
+ <a href="/">
+ <span class="sr-only">Apache ORC</span>
+ <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
+ </a>
+ </h1>
+ </div>
+ <nav class="main-nav unit two-thirds hide-on-mobiles">
+ <ul>
+ <li class="">
+ <a href="/">Home</a>
+ </li>
+ <li class="current">
+ <a href="/docs/">Doc<span class="show-on-mobiles">s</span>
+ <span class="hide-on-mobiles">umentation</span></a>
+ </li>
+ <li class="">
+ <a href="/talks/">Talks</a>
+ </li>
+ <li class="">
+ <a href="/news/">News</a>
+ </li>
+ <li class="">
+ <a href="/help/">Help</a>
+ </li>
+ <li class="">
+ <a href="/develop/">Develop</a>
+ </li>
+</ul>
+
+ </nav>
+ </div>
+</header>
+
+
+ <section class="docs">
+ <div class="grid">
+
+ <div class="docs-nav-mobile unit whole show-on-mobiles">
+ <select onchange="if (this.value) window.location.href=this.value">
+ <option value="">Navigate the docs\u2026</option>
+
+ <optgroup label="Overview">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/index.html">Background</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/types.html">Types</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/indexes.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/acid.html">ACID support</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Installing">
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/building.html">Building ORC</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/releases.html">Releases</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in Hive">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using ORC Core">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/core-java.html">Using Core Java</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Tools">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/tools.html">Tools</option>
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Format Specification">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Releases</h1>
+ <h2 id="current-release---110">Current Release - 1.1.0:</h2>
+
+<p>ORC 1.1.0 contains both the Java reader and writer and the C++ reader. It also
+contains tools for working with ORC files and looking at their contents and
+metadata.</p>
+
+<ul>
+ <li>Released: 10 Jun 2016</li>
+ <li>Source code: <a href="http://www.apache.org/dyn/closer.cgi/orc/orc-1.1.0/orc-1.1.0.tgz">orc-1.1.0.tgz</a></li>
+ <li><a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.asc">GPG Signature</a>
+signed by <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">Owen O\u2019Malley (3D0C92B9)</a></li>
+ <li>Git tag: <a href="https://github.com/apache/orc/releases/tag/rel/release-1.1.0">fc571e77</a></li>
+ <li>SHA 256: <a href="https://dist.apache.org/repos/dist/release/orc/orc-1.1.0/orc-1.1.0.tgz.sha256">8beea2be</a></li>
+</ul>
+
+<p>Known issues:</p>
+
+<ul>
+ <li><a href="https://issues.apache.org/jira/browse/ORC-40">ORC-40</a> Predicate push down is not implemented in C++.</li>
+</ul>
+
+<h2 id="checking-signatures">Checking signatures</h2>
+
+<p>All GPG signatures should be verified as matching one of the keys in ORC\u2019s
+committers\u2019 <a href="https://dist.apache.org/repos/dist/release/orc/KEYS">key list</a>.</p>
+
+<pre><code class="language-shell">% shasum -a 256 orc-1.1.0.tgz | diff - orc-1.1.0.tgz.sha256
+% gpg --import KEYS
+% gpg --verify orc-1.1.0.tgz.asc
+</code></pre>
+
+<h2 id="previous-releases">Previous releases:</h2>
+
+<table>
+ <thead>
+ <tr>
+ <th style="text-align: center">Version</th>
+ <th style="text-align: center">Date</th>
+ <th style="text-align: center">Release</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td style="text-align: center">1.0.0</td>
+ <td style="text-align: center">25 Jan 2016</td>
+ <td style="text-align: center"><a href="/news/2016/01/25/ORC-1.0.0/">ORC-1.0.0</a></td>
+ </tr>
+ </tbody>
+</table>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/building.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/hive-ddl.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class="current"><a href="/docs/releases.html">Releases</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in Hive</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+
+
+
+</ul>
+
+
+ <h4>Tools</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
+
+
+
+</ul>
+
+
+ <h4>Format Specification</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/compression.html">Compression</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+
+
+
+</ul>
+
+
+ </aside>
+</div>
+
+
+ <div class="clear"></div>
+
+ </div>
+ </section>
+
+
+ <footer role="contentinfo">
+ <p>The contents of this website are © 2016
+ <a href="https://www.apache.org/">Apache Software Foundation</a>
+ under the terms of the <a
+ href="https://www.apache.org/licenses/LICENSE-2.0.html">
+ Apache License v2</a>. Apache ORC and its logo are trademarks
+ of the Apache Software Foundation.</p>
+</footer>
+
+ <script>
+ var anchorForId = function (id) {
+ var anchor = document.createElement("a");
+ anchor.className = "header-link";
+ anchor.href = "#" + id;
+ anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
+ anchor.title = "Permalink";
+ return anchor;
+ };
+
+ var linkifyAnchors = function (level, containingElement) {
+ var headers = containingElement.getElementsByTagName("h" + level);
+ for (var h = 0; h < headers.length; h++) {
+ var header = headers[h];
+
+ if (typeof header.id !== "undefined" && header.id !== "") {
+ header.appendChild(anchorForId(header.id));
+ }
+ }
+ };
+
+ document.onreadystatechange = function () {
+ if (this.readyState === "complete") {
+ var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
+ if (!contentBlock) {
+ return;
+ }
+ for (var level = 1; level <= 6; level++) {
+ linkifyAnchors(level, contentBlock);
+ }
+ }
+ };
+</script>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/run-length.html
----------------------------------------------------------------------
diff --git a/docs/run-length.html b/docs/run-length.html
index ee63252..71f620d 100644
--- a/docs/run-length.html
+++ b/docs/run-length.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
+ <optgroup label="Format Specification">
+
- <div class="unit four-fifths">
- <article>
- <h1>Run Length Encoding</h1>
- <h1 id="base-128-varint">Base 128 Varint</h1>
-<p>Variable width integer encodings take advantage of the fact that most
-numbers are small and that having smaller encodings for small numbers
-shrinks the overall size of the data. ORC uses the varint format from
-Protocol Buffers, which writes data in little endian format using the
-low 7 bits of each byte. The high bit in each byte is set if the
-number continues into the next byte.</p>
+
-<table>
- <thead>
- <tr>
- <th style="text-align: left">Unsigned Original</th>
- <th style="text-align: left">Serialized</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td style="text-align: left">0</td>
- <td style="text-align: left">0x00</td>
- </tr>
- <tr>
- <td style="text-align: left">1</td>
- <td style="text-align: left">0x01</td>
- </tr>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Run Length Encoding</h1>
+ <h1 id="base-128-varint">Base 128 Varint</h1>
+
+<p>Variable width integer encodings take advantage of the fact that most
+numbers are small and that having smaller encodings for small numbers
+shrinks the overall size of the data. ORC uses the varint format from
+Protocol Buffers, which writes data in little endian format using the
+low 7 bits of each byte. The high bit in each byte is set if the
+number continues into the next byte.</p>
+
+<table>
+ <thead>
+ <tr>
+ <th style="text-align: left">Unsigned Original</th>
+ <th style="text-align: left">Serialized</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td style="text-align: left">0</td>
+ <td style="text-align: left">0x00</td>
+ </tr>
+ <tr>
+ <td style="text-align: left">1</td>
+ <td style="text-align: left">0x01</td>
+ </tr>
<tr>
<td style="text-align: left">127</td>
<td style="text-align: left">0x7f</td>
@@ -897,95 +1429,324 @@ base value is 2000 and the combined result is [0x8e, 0x09, 0x2b, 0x21,
0x07, 0xd0, 0x1e, 0x00, 0x14, 0x70, 0x28, 0x32, 0x3c, 0x46, 0x50,
0x5a, 0xfc, 0xe8]</p>
-<h2 id="delta">Delta</h2>
+<h2 id="delta">Delta</h2>
+
+<p>The Delta encoding is used for monotonically increasing or decreasing
+sequences. The first two numbers in the sequence can not be identical,
+because the encoding is using the sign of the first delta to determine
+if the series is increasing or decreasing.</p>
+
+<ul>
+ <li>2 bytes header
+ <ul>
+ <li>2 bits for encoding type (3)</li>
+ <li>5 bits for encoded width (W) of deltas (0 to 64 bits) using the 5 bit
+width encoding table</li>
+ <li>9 bits for run length (L) (1 to 512 values)</li>
+ </ul>
+ </li>
+ <li>Base value - encoded as (signed or unsigned) varint</li>
+ <li>Delta base - encoded as signed varint</li>
+ <li>Delta values $W * (L - 2)$ bytes - encode each delta after the first
+one. If the delta base is positive, the sequence is increasing and if it is
+negative the sequence is decreasing.</li>
+</ul>
+
+<p>The unsigned sequence of [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] would be
+serialized with delta encoding (3), a width of 4 bits (3), length of
+10 (9), a base of 2 (2), and first delta of 1 (2). The resulting
+sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/compression.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/stripes.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
-<p>The Delta encoding is used for monotonically increasing or decreasing
-sequences. The first two numbers in the sequence can not be identical,
-because the encoding is using the sign of the first delta to determine
-if the series is increasing or decreasing.</p>
+
-<ul>
- <li>2 bytes header
- <ul>
- <li>2 bits for encoding type (3)</li>
- <li>5 bits for encoded width (W) of deltas (0 to 64 bits) using the 5 bit
-width encoding table</li>
- <li>9 bits for run length (L) (1 to 512 values)</li>
- </ul>
- </li>
- <li>Base value - encoded as (signed or unsigned) varint</li>
- <li>Delta base - encoded as signed varint</li>
- <li>Delta values $W * (L - 2)$ bytes - encode each delta after the first
-one. If the delta base is positive, the sequence is increasing and if it is
-negative the sequence is decreasing.</li>
-</ul>
+
+
+
-<p>The unsigned sequence of [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] would be
-serialized with delta encoding (3), a width of 4 bits (3), length of
-10 (9), a base of 2 (2), and first delta of 1 (2). The resulting
-sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
-
+</ul>
+
+ <h4>Installing</h4>
+
+<ul>
+
+
+
+
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
-
+
+
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/compression.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/stripes.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -1010,7 +1771,11 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -1036,6 +1801,38 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1046,7 +1843,7 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -1072,10 +1869,28 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -1084,14 +1899,22 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -1114,16 +1937,12 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -1134,7 +1953,17 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -1174,6 +2003,18 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -1192,6 +2033,12 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -1206,6 +2053,10 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -1234,6 +2085,18 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class="current"><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -1268,6 +2131,18 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -1284,6 +2159,12 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1314,6 +2195,18 @@ sequence is [0xc6, 0x09, 0x02, 0x02, 0x22, 0x42, 0x42, 0x46].</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-index.html">Indexes</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/1eb37b72/docs/spec-index.html
----------------------------------------------------------------------
diff --git a/docs/spec-index.html b/docs/spec-index.html
index ee509cf..a1ac5c9 100644
--- a/docs/spec-index.html
+++ b/docs/spec-index.html
@@ -105,6 +105,12 @@
+
+
+
+
+
+
<option value="/docs/index.html">Background</option>
@@ -120,6 +126,60 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/adopters.html">ORC Adopters</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -149,6 +209,20 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
<option value="/docs/types.html">Types</option>
@@ -171,6 +245,12 @@
+
+
+
+
+
+
<option value="/docs/indexes.html">Indexes</option>
@@ -184,6 +264,14 @@
+
+
+
+
+
+
+
+
@@ -216,11 +304,25 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
</optgroup>
- <optgroup label="Hive Usage">
+ <optgroup label="Installing">
@@ -232,13 +334,27 @@
+ <option value="/docs/building.html">Building ORC</option>
+
- <option value="/docs/hive-ddl.html">Hive DDL</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -268,7 +384,17 @@
- <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
+
+
+
+
+
+
+
+
+
@@ -276,6 +402,10 @@
+ <option value="/docs/releases.html">Releases</option>
+
+
+
@@ -291,7 +421,7 @@
</optgroup>
- <optgroup label="Format Specification">
+ <optgroup label="Using in Hive">
@@ -315,11 +445,25 @@
+ <option value="/docs/hive-ddl.html">Hive DDL</option>
+
- <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -337,7 +481,17 @@
- <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+ <option value="/docs/hive-config.html">Hive Configuration</option>
+
+
@@ -358,6 +512,17 @@
+
+
+
+
+
+
+ </optgroup>
+
+ <optgroup label="Using in MapReduce">
+
+
@@ -365,7 +530,11 @@
- <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
@@ -381,6 +550,16 @@
+ <option value="/docs/mapred.html">Using in MapRed</option>
+
+
+
+
+
+
+
+
+
@@ -411,7 +590,7 @@
- <option value="/docs/run-length.html">Run Length Encoding</option>
+
@@ -419,12 +598,14 @@
+ <option value="/docs/mapreduce.html">Using in MapReduce</option>
+
-
+
-
+
@@ -434,7 +615,16 @@
+
+
+ </optgroup>
+ <optgroup label="Using ORC Core">
+
+
+
+
+
@@ -445,25 +635,27 @@
+ <option value="/docs/core-java.html">Using Core Java</option>
+
- <option value="/docs/stripes.html">Stripes</option>
+
-
+
-
+
- <option value="/docs/encodings.html">Column Encodings</option>
+
@@ -476,7 +668,16 @@
+
+
+ </optgroup>
+ <optgroup label="Tools">
+
+
+
+
+
@@ -486,9 +687,9 @@
-
+
-
+
@@ -509,12 +710,14 @@
- <option value="/docs/spec-index.html">Indexes</option>
+
+ <option value="/docs/tools.html">Tools</option>
+
@@ -522,38 +725,367 @@
</optgroup>
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Indexes</h1>
- <h1 id="row-group-index">Row Group Index</h1>
-
-<p>The row group indexes consist of a ROW_INDEX stream for each primitive
-column that has an entry for each row group. Row groups are controlled
-by the writer and default to 10,000 rows. Each RowIndexEntry gives the
-position of each stream for the column and the statistics for that row
-group.</p>
-
-<p>The index streams are placed at the front of the stripe, because in
-the default case of streaming they do not need to be read. They are
-only loaded when either predicate push down is being used or the
-reader seeks to a particular row.</p>
+ <optgroup label="Format Specification">
+
-<p><code>message RowIndexEntry {
- repeated uint64 positions = 1 [packed=true];
- optional ColumnStatistics statistics = 2;
-}
-</code></p>
-<p><code>message RowIndex {
- repeated RowIndexEntry entry = 1;
-}
-</code></p>
+
-<p>To record positions, each stream needs a sequence of numbers. For
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-intro.html">Introduction</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/file-tail.html">File Tail</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/compression.html">Compression</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/run-length.html">Run Length Encoding</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/stripes.html">Stripes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/encodings.html">Column Encodings</option>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <option value="/docs/spec-index.html">Indexes</option>
+
+
+
+
+
+
+
+
+
+
+
+
+ </optgroup>
+
+ </select>
+</div>
+
+
+ <div class="unit four-fifths">
+ <article>
+ <h1>Indexes</h1>
+ <h1 id="row-group-index">Row Group Index</h1>
+
+<p>The row group indexes consist of a ROW_INDEX stream for each primitive
+column that has an entry for each row group. Row groups are controlled
+by the writer and default to 10,000 rows. Each RowIndexEntry gives the
+position of each stream for the column and the statistics for that row
+group.</p>
+
+<p>The index streams are placed at the front of the stripe, because in
+the default case of streaming they do not need to be read. They are
+only loaded when either predicate push down is being used or the
+reader seeks to a particular row.</p>
+
+<p><code>message RowIndexEntry {
+ repeated uint64 positions = 1 [packed=true];
+ optional ColumnStatistics statistics = 2;
+}
+</code></p>
+
+<p><code>message RowIndex {
+ repeated RowIndexEntry entry = 1;
+}
+</code></p>
+
+<p>To record positions, each stream needs a sequence of numbers. For
uncompressed streams, the position is the byte offset of the RLE run\u2019s
start location followed by the number of values that need to be
consumed from the run. In compressed streams, the first number is the
@@ -647,76 +1179,305 @@ within bitset and bit position within the long uses little endian order.
makes it convenient to read the bloom filter stream and row index stream
together in single read operation.</p>
-<p><img src="/img/BloomFilter.png" alt="bloom filter" /></p>
+<p><img src="/img/BloomFilter.png" alt="bloom filter" /></p>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/encodings.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+ <span class="next disabled">Next</span>
+
+ </div>
+ </div>
+ <div class="clear"></div>
+
+
+ </article>
+ </div>
+
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
+
+ <h4>Overview</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/types.html">Types</a></li>
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+</ul>
+
+
+ <h4>Installing</h4>
+
+
+<ul>
+
+
+
-
+
+
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
+
+
+
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
-
+
+
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/encodings.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
- <span class="next disabled">Next</span>
-
- </div>
- </div>
- <div class="clear"></div>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
+
- </article>
- </div>
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
+</ul>
+
- <h4>Overview</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -741,7 +1502,11 @@ together in single read operation.</p>
- <li class=""><a href="/docs/index.html">Background</a></li>
+
+
+
+
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -767,6 +1532,38 @@ together in single read operation.</p>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+</ul>
+
+
+ <h4>Using in MapReduce</h4>
+
+
+<ul>
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -777,7 +1574,7 @@ together in single read operation.</p>
- <li class=""><a href="/docs/types.html">Types</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -803,10 +1600,28 @@ together in single read operation.</p>
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+</ul>
+
+
+ <h4>Using ORC Core</h4>
+
+
+<ul>
+
@@ -815,14 +1630,22 @@ together in single read operation.</p>
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
</ul>
- <h4>Hive Usage</h4>
+ <h4>Tools</h4>
<ul>
@@ -845,16 +1668,12 @@ together in single read operation.</p>
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
+
-
+
@@ -865,7 +1684,17 @@ together in single read operation.</p>
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li class=""><a href="/docs/tools.html">Tools</a></li>
@@ -905,6 +1734,18 @@ together in single read operation.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
@@ -923,6 +1764,12 @@ together in single read operation.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/file-tail.html">File Tail</a></li>
@@ -937,6 +1784,10 @@ together in single read operation.</p>
+
+
+
+
<li class=""><a href="/docs/compression.html">Compression</a></li>
@@ -965,6 +1816,18 @@ together in single read operation.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
@@ -999,6 +1862,18 @@ together in single read operation.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class=""><a href="/docs/stripes.html">Stripes</a></li>
@@ -1015,6 +1890,12 @@ together in single read operation.</p>
+
+
+
+
+
+
<li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
@@ -1045,6 +1926,18 @@ together in single read operation.</p>
+
+
+
+
+
+
+
+
+
+
+
+
<li class="current"><a href="/docs/spec-index.html">Indexes</a></li>