You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2018/04/17 17:49:56 UTC
[9/9] orc git commit: Pushing ORC-339 reorganize the ORC file format
spec.
Pushing ORC-339 reorganize the ORC file format spec.
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/orc/repo
Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/c6e29090
Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/c6e29090
Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/c6e29090
Branch: refs/heads/asf-site
Commit: c6e2909025381446398961f4ac1da61550cd13b5
Parents: c63412b
Author: Owen O'Malley <om...@apache.org>
Authored: Tue Apr 17 10:49:12 2018 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Tue Apr 17 10:49:12 2018 -0700
----------------------------------------------------------------------
develop/index.html | 3 +
docs/acid.html | 1073 ++--------------
docs/adopters.html | 1185 ++---------------
docs/building.html | 1071 ++--------------
docs/compression.html | 2193 --------------------------------
docs/core-cpp.html | 1429 ++++-----------------
docs/core-java.html | 1083 ++--------------
docs/cpp-tools.html | 1523 +++++-----------------
docs/encodings.html | 2790 -----------------------------------------
docs/file-tail.html | 2477 ------------------------------------
docs/hive-config.html | 1075 ++--------------
docs/hive-ddl.html | 1145 ++---------------
docs/index.html | 1329 +++-----------------
docs/indexes.html | 1133 ++---------------
docs/java-tools.html | 1549 +++++------------------
docs/mapred.html | 1083 ++--------------
docs/mapreduce.html | 1085 ++--------------
docs/releases.html | 1071 ++--------------
docs/run-length.html | 2566 -------------------------------------
docs/spec-index.html | 2298 ---------------------------------
docs/spec-intro.html | 2180 --------------------------------
docs/stripes.html | 2257 ---------------------------------
docs/types.html | 1215 +++---------------
specification/ORCv0.html | 1260 +++++++++++++++++++
specification/ORCv1.html | 1744 ++++++++++++++++++++++++++
specification/ORCv2.html | 1769 ++++++++++++++++++++++++++
specification/index.html | 159 +++
27 files changed, 7126 insertions(+), 32619 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/develop/index.html
----------------------------------------------------------------------
diff --git a/develop/index.html b/develop/index.html
index e920320..d9224d0 100644
--- a/develop/index.html
+++ b/develop/index.html
@@ -87,6 +87,9 @@
<p>Information about the ORC project that is most important for
developers working on the project.</p>
+<p>The <a href="/specification">ORC format specification</a> defines the format
+to promote compatibility between implementations.</p>
+
<h2 id="development-community">Development community</h2>
<p>We have committers from many different companies. The full
http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/acid.html
----------------------------------------------------------------------
diff --git a/docs/acid.html b/docs/acid.html
index c460d41..71c980c 100644
--- a/docs/acid.html
+++ b/docs/acid.html
@@ -109,12 +109,6 @@
-
-
-
-
-
-
<option value="/docs/index.html">Background</option>
@@ -130,14 +124,6 @@
-
-
-
-
-
-
-
-
@@ -174,20 +160,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -221,20 +193,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
<option value="/docs/types.html">Types</option>
@@ -261,12 +219,6 @@
-
-
-
-
-
-
<option value="/docs/indexes.html">Indexes</option>
@@ -280,14 +232,6 @@
-
-
-
-
-
-
-
-
@@ -324,20 +268,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
</optgroup>
@@ -381,20 +311,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -426,25 +342,11 @@
-
-
-
-
-
-
<option value="/docs/releases.html">Releases</option>
-
-
-
-
-
-
-
-
</optgroup>
@@ -471,12 +373,6 @@
-
-
-
-
-
-
<option value="/docs/hive-ddl.html">Hive DDL</option>
@@ -494,14 +390,6 @@
-
-
-
-
-
-
-
-
@@ -519,12 +407,6 @@
-
-
-
-
-
-
<option value="/docs/hive-config.html">Hive Configuration</option>
@@ -544,14 +426,6 @@
-
-
-
-
-
-
-
-
</optgroup>
@@ -586,12 +460,6 @@
-
-
-
-
-
-
<option value="/docs/mapred.html">Using in MapRed</option>
@@ -601,14 +469,6 @@
-
-
-
-
-
-
-
-
@@ -638,12 +498,6 @@
-
-
-
-
-
-
<option value="/docs/mapreduce.html">Using in MapReduce</option>
@@ -651,14 +505,6 @@
-
-
-
-
-
-
-
-
</optgroup>
@@ -679,8 +525,6 @@
-
-
<option value="/docs/core-java.html">Using Core Java</option>
@@ -704,18 +548,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -727,8 +559,6 @@
-
-
<option value="/docs/core-cpp.html">Using Core C++</option>
@@ -754,18 +584,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
</optgroup>
@@ -788,8 +606,6 @@
-
-
<option value="/docs/cpp-tools.html">C++ Tools</option>
@@ -811,18 +627,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -848,12 +652,6 @@
-
-
-
-
-
-
<option value="/docs/java-tools.html">Java Tools</option>
@@ -865,386 +663,21 @@
-
-
-
-
-
-
-
-
</optgroup>
- <optgroup label="Format Specification">
-
+ </select>
+</div>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/file-tail.html">File Tail</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/compression.html">Compression</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/stripes.html">Stripes</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/encodings.html">Column Encodings</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-index.html">Indexes</option>
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>ACID support</h1>
- <p>Historically, the only way to atomically add data to a table in Hive
-was to add a new partition. Updating or deleting data in partition
-required removing the old partition and adding it back with the new
-data and it wasn’t possible to do atomically.</p>
+ <div class="unit four-fifths">
+ <article>
+ <h1>ACID support</h1>
+ <p>Historically, the only way to atomically add data to a table in Hive
+was to add a new partition. Updating or deleting data in partition
+required removing the old partition and adding it back with the new
+data and it wasn’t possible to do atomically.</p>
<p>However, user’s data is continually changing and as Hive matured,
users required reliability guarantees despite the churning data
@@ -1465,270 +898,44 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/indexes.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/building.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
-
-
- </article>
- </div>
-
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
-
-
-
-
-
- <li class="current"><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
-
- <h4>Installing</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/building.html">Building ORC</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
+
-
-
+
-
-
+
-
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/indexes.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/building.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
- <li class=""><a href="/docs/releases.html">Releases</a></li>
-
+ </article>
+ </div>
-</ul>
-
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
- <h4>Using in Hive</h4>
+ <h4>Overview</h4>
<ul>
@@ -1757,11 +964,7 @@ file that don’t need to be read in this task.</p>
-
-
-
-
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <li class=""><a href="/docs/index.html">Background</a></li>
@@ -1775,34 +978,10 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
-</ul>
-
-
- <h4>Using in MapReduce</h4>
-
-
-<ul>
-
@@ -1839,7 +1018,7 @@ file that don’t need to be read in this task.</p>
- <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+ <li class=""><a href="/docs/types.html">Types</a></li>
@@ -1869,49 +1048,7 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
-
-
-
-</ul>
-
-
- <h4>Using ORC Core</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
@@ -1923,22 +1060,14 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
+ <li class="current"><a href="/docs/acid.html">ACID support</a></li>
</ul>
- <h4>Tools</h4>
+ <h4>Installing</h4>
<ul>
@@ -1955,15 +1084,7 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
@@ -2001,14 +1122,14 @@ file that don’t need to be read in this task.</p>
- <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
</ul>
- <h4>Format Specification</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -2035,31 +1156,7 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -2083,31 +1180,17 @@ file that don’t need to be read in this task.</p>
-
-
-
-
- <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-
-
+</ul>
-
-
-
-
-
-
+ <h4>Using in MapReduce</h4>
- <li class=""><a href="/docs/compression.html">Compression</a></li>
-
+<ul>
@@ -2139,19 +1222,7 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -2187,13 +1258,25 @@ file that don’t need to be read in this task.</p>
-
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
-
+ <h4>Using ORC Core</h4>
+
+<ul>
+
+
+
+
+
@@ -2203,7 +1286,7 @@ file that don’t need to be read in this task.</p>
- <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
@@ -2221,17 +1304,17 @@ file that don’t need to be read in this task.</p>
-
-
-
-
-
+ <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
+
+
+
+</ul>
+
-
+ <h4>Tools</h4>
- <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
-
+<ul>
@@ -2251,11 +1334,17 @@ file that don’t need to be read in this task.</p>
+ <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
+
+
+
-
+
+
+
@@ -2277,7 +1366,7 @@ file that don’t need to be read in this task.</p>
- <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+ <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/adopters.html
----------------------------------------------------------------------
diff --git a/docs/adopters.html b/docs/adopters.html
index b30ef6e..7ee402b 100644
--- a/docs/adopters.html
+++ b/docs/adopters.html
@@ -109,12 +109,6 @@
-
-
-
-
-
-
<option value="/docs/index.html">Background</option>
@@ -130,14 +124,6 @@
-
-
-
-
-
-
-
-
@@ -174,20 +160,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -221,20 +193,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
<option value="/docs/types.html">Types</option>
@@ -261,12 +219,6 @@
-
-
-
-
-
-
<option value="/docs/indexes.html">Indexes</option>
@@ -280,14 +232,6 @@
-
-
-
-
-
-
-
-
@@ -324,20 +268,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
</optgroup>
@@ -381,20 +311,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -426,25 +342,11 @@
-
-
-
-
-
-
<option value="/docs/releases.html">Releases</option>
-
-
-
-
-
-
-
-
</optgroup>
@@ -471,12 +373,6 @@
-
-
-
-
-
-
<option value="/docs/hive-ddl.html">Hive DDL</option>
@@ -494,14 +390,6 @@
-
-
-
-
-
-
-
-
@@ -519,12 +407,6 @@
-
-
-
-
-
-
<option value="/docs/hive-config.html">Hive Configuration</option>
@@ -544,14 +426,6 @@
-
-
-
-
-
-
-
-
</optgroup>
@@ -586,12 +460,6 @@
-
-
-
-
-
-
<option value="/docs/mapred.html">Using in MapRed</option>
@@ -601,14 +469,6 @@
-
-
-
-
-
-
-
-
@@ -638,12 +498,6 @@
-
-
-
-
-
-
<option value="/docs/mapreduce.html">Using in MapReduce</option>
@@ -651,14 +505,6 @@
-
-
-
-
-
-
-
-
</optgroup>
@@ -679,8 +525,6 @@
-
-
<option value="/docs/core-java.html">Using Core Java</option>
@@ -704,18 +548,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -727,8 +559,6 @@
-
-
<option value="/docs/core-cpp.html">Using Core C++</option>
@@ -754,18 +584,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
</optgroup>
@@ -788,8 +606,6 @@
-
-
<option value="/docs/cpp-tools.html">C++ Tools</option>
@@ -811,18 +627,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -848,12 +652,6 @@
-
-
-
-
-
-
<option value="/docs/java-tools.html">Java Tools</option>
@@ -865,717 +663,126 @@
-
-
-
-
-
-
-
-
</optgroup>
- <optgroup label="Format Specification">
-
+ </select>
+</div>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/file-tail.html">File Tail</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/compression.html">Compression</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/stripes.html">Stripes</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/encodings.html">Column Encodings</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-index.html">Indexes</option>
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>ORC Adopters</h1>
- <p>If your company or tool uses ORC, please let us know so that we can update
-this page.</p>
-
-<h3 id="apache-hadoophttpshadoopapacheorg"><a href="https://hadoop.apache.org/">Apache Hadoop</a></h3>
-
-<p>ORC files have always supporting reading and writing from Hadoop’s MapReduce,
-but with the ORC 1.1.0 release it is now easier than ever without pulling in
-Hive’s exec jar and all of its dependencies. OrcStruct now also implements
-WritableComparable and can be serialized through the MapReduce shuffle.</p>
-
-<h3 id="apache-hivehttpshiveapacheorg"><a href="https://hive.apache.org/">Apache Hive</a></h3>
-
-<p>Apache Hive was the original use case and home for ORC. ORC’s strong
-type system, advanced compression, column projection, predicate push
-down, and vectorization support make Hive <a href="https://hortonworks.com/blog/orcfile-in-hdp-2-better-compression-better-performance/">perform
-better</a>
-than any other format for your data.</p>
-
-<h3 id="apache-nifihttpsnifiapacheorg"><a href="https://nifi.apache.org/">Apache Nifi</a></h3>
-
-<p>Apache Nifi is <a href="https://issues.apache.org/jira/browse/NIFI-1663">adding
-support</a> for writing
-ORC files.</p>
-
-<h3 id="apache-pighttpspigapacheorg"><a href="https://pig.apache.org/">Apache Pig</a></h3>
-
-<p>Apache Pig added support for reading and writing ORC files in <a href="https://hortonworks.com/blog/announcing-apache-pig-0-14-0/">Pig
-14.0</a>.</p>
-
-<h3 id="apache-sparkhttpssparkapacheorg"><a href="https://spark.apache.org/">Apache Spark</a></h3>
-
-<p>Apache Spark has <a href="https://hortonworks.com/blog/bringing-orc-support-into-apache-spark/">added
-support</a>
-for reading and writing ORC files with support for column project and
-predicate push down.</p>
-
-<h3 id="eelhttpsgithubcom51zeroeel-sdk"><a href="https://github.com/51zero/eel-sdk">EEL</a></h3>
-
-<p>EEL is a Scala BigData API that supports reading and writing data for
-various file formats and storage systems including to and from ORC. It
-is designed as a in-process low level API for manipulating data. Data
-is lazily streamed from source to sink and using standard Scala
-operations such as map, flatMap and filter, it is especially suited
-for ETL style applications. EEL supports ORC predicate and projection
-pushdowns and correct handles conversions from other formats including
-complex types such as maps, lists or nested structs. A typical use
-case would be to extract data from JDBC to ORC files housed in HDFS,
-or directly into Hive tables backed by an ORC file format.</p>
-
-<h3 id="facebookhttpsfacebookcom"><a href="https://facebook.com">Facebook</a></h3>
-
-<p>With more than 300 PB of data, Facebook was an <a href="https://code.facebook.com/posts/229861827208629/scaling-the-facebook-data-warehouse-to-300-pb/">early adopter of
-ORC</a> and quickly put it into production.</p>
-
-<h3 id="prestohttpsprestodbio"><a href="https://prestodb.io/">Presto</a></h3>
-
-<p>The Presto team has done a lot of work <a href="https://code.facebook.com/posts/370832626374903/even-faster-data-at-the-speed-of-presto-orc/">integrating
-ORC</a> into their SQL engine.</p>
-
-<h3 id="timberhttpstimberio"><a href="https://timber.io/">Timber</a></h3>
-
-<p>Timber adopted ORC for it’s S3 based logging platform that stores
-petabytes of log data. ORC has been key in ensuring a fast,
-cost-effective strategy for persisting and querying that data.</p>
-
-<h3 id="verticahttpwww8hpcomusensoftware-solutionsadvanced-sql-big-data-analytics"><a href="http://www8.hp.com/us/en/software-solutions/advanced-sql-big-data-analytics/">Vertica</a></h3>
-
-<p>HPE Vertica has contributed significantly to the ORC C++ library. ORC
-is a significant part of Vertica SQL-on-Hadoop (VSQLoH) which brings
-the performance, reliability and standards compliance of the Vertica
-Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/index.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/types.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
-
-
- </article>
- </div>
-
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class="current"><a href="/docs/adopters.html">ORC Adopters</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
+ <div class="unit four-fifths">
+ <article>
+ <h1>ORC Adopters</h1>
+ <p>If your company or tool uses ORC, please let us know so that we can update
+this page.</p>
+<h3 id="apache-hadoophttpshadoopapacheorg"><a href="https://hadoop.apache.org/">Apache Hadoop</a></h3>
-
+<p>ORC files have always supporting reading and writing from Hadoop’s MapReduce,
+but with the ORC 1.1.0 release it is now easier than ever without pulling in
+Hive’s exec jar and all of its dependencies. OrcStruct now also implements
+WritableComparable and can be serialized through the MapReduce shuffle.</p>
-
-
-
+<h3 id="apache-hivehttpshiveapacheorg"><a href="https://hive.apache.org/">Apache Hive</a></h3>
-
-
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
+<p>Apache Hive was the original use case and home for ORC. ORC’s strong
+type system, advanced compression, column projection, predicate push
+down, and vectorization support make Hive <a href="https://hortonworks.com/blog/orcfile-in-hdp-2-better-compression-better-performance/">perform
+better</a>
+than any other format for your data.</p>
+<h3 id="apache-nifihttpsnifiapacheorg"><a href="https://nifi.apache.org/">Apache Nifi</a></h3>
-</ul>
+<p>Apache Nifi is <a href="https://issues.apache.org/jira/browse/NIFI-1663">adding
+support</a> for writing
+ORC files.</p>
-
- <h4>Installing</h4>
-
+<h3 id="apache-pighttpspigapacheorg"><a href="https://pig.apache.org/">Apache Pig</a></h3>
-<ul>
+<p>Apache Pig added support for reading and writing ORC files in <a href="https://hortonworks.com/blog/announcing-apache-pig-0-14-0/">Pig
+14.0</a>.</p>
-
+<h3 id="apache-sparkhttpssparkapacheorg"><a href="https://spark.apache.org/">Apache Spark</a></h3>
-
-
-
+<p>Apache Spark has <a href="https://hortonworks.com/blog/bringing-orc-support-into-apache-spark/">added
+support</a>
+for reading and writing ORC files with support for column project and
+predicate push down.</p>
+
+<h3 id="eelhttpsgithubcom51zeroeel-sdk"><a href="https://github.com/51zero/eel-sdk">EEL</a></h3>
+
+<p>EEL is a Scala BigData API that supports reading and writing data for
+various file formats and storage systems including to and from ORC. It
+is designed as a in-process low level API for manipulating data. Data
+is lazily streamed from source to sink and using standard Scala
+operations such as map, flatMap and filter, it is especially suited
+for ETL style applications. EEL supports ORC predicate and projection
+pushdowns and correct handles conversions from other formats including
+complex types such as maps, lists or nested structs. A typical use
+case would be to extract data from JDBC to ORC files housed in HDFS,
+or directly into Hive tables backed by an ORC file format.</p>
+
+<h3 id="facebookhttpsfacebookcom"><a href="https://facebook.com">Facebook</a></h3>
+
+<p>With more than 300 PB of data, Facebook was an <a href="https://code.facebook.com/posts/229861827208629/scaling-the-facebook-data-warehouse-to-300-pb/">early adopter of
+ORC</a> and quickly put it into production.</p>
+
+<h3 id="prestohttpsprestodbio"><a href="https://prestodb.io/">Presto</a></h3>
+
+<p>The Presto team has done a lot of work <a href="https://code.facebook.com/posts/370832626374903/even-faster-data-at-the-speed-of-presto-orc/">integrating
+ORC</a> into their SQL engine.</p>
+
+<h3 id="timberhttpstimberio"><a href="https://timber.io/">Timber</a></h3>
+
+<p>Timber adopted ORC for it’s S3 based logging platform that stores
+petabytes of log data. ORC has been key in ensuring a fast,
+cost-effective strategy for persisting and querying that data.</p>
+
+<h3 id="verticahttpwww8hpcomusensoftware-solutionsadvanced-sql-big-data-analytics"><a href="http://www8.hp.com/us/en/software-solutions/advanced-sql-big-data-analytics/">Vertica</a></h3>
+
+<p>HPE Vertica has contributed significantly to the ORC C++ library. ORC
+is a significant part of Vertica SQL-on-Hadoop (VSQLoH) which brings
+the performance, reliability and standards compliance of the Vertica
+Analytic Database to the Hadoop ecosystem.</p>
+
+
-
-
-
-
-
-
- <li class=""><a href="/docs/building.html">Building ORC</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/index.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/types.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
- <li class=""><a href="/docs/releases.html">Releases</a></li>
-
+ </article>
+ </div>
-</ul>
-
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
- <h4>Using in Hive</h4>
+ <h4>Overview</h4>
<ul>
@@ -1604,11 +811,7 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <li class=""><a href="/docs/index.html">Background</a></li>
@@ -1622,34 +825,10 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+ <li class="current"><a href="/docs/adopters.html">ORC Adopters</a></li>
-</ul>
-
-
- <h4>Using in MapReduce</h4>
-
-
-<ul>
-
@@ -1686,7 +865,7 @@ Analytic Database to the Hadoop ecosystem.</p>
- <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+ <li class=""><a href="/docs/types.html">Types</a></li>
@@ -1716,49 +895,7 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
-
-
-
-</ul>
-
-
- <h4>Using ORC Core</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
@@ -1770,22 +907,14 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
</ul>
- <h4>Tools</h4>
+ <h4>Installing</h4>
<ul>
@@ -1802,15 +931,7 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
+ <li class=""><a href="/docs/building.html">Building ORC</a></li>
@@ -1848,14 +969,14 @@ Analytic Database to the Hadoop ecosystem.</p>
- <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
</ul>
- <h4>Format Specification</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -1882,31 +1003,7 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -1930,31 +1027,17 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
- <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-
-
+</ul>
-
-
-
-
-
-
+ <h4>Using in MapReduce</h4>
- <li class=""><a href="/docs/compression.html">Compression</a></li>
-
+<ul>
@@ -1986,19 +1069,7 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -2034,13 +1105,25 @@ Analytic Database to the Hadoop ecosystem.</p>
-
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
-
+ <h4>Using ORC Core</h4>
+
+<ul>
+
+
+
+
+
@@ -2050,7 +1133,7 @@ Analytic Database to the Hadoop ecosystem.</p>
- <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
@@ -2068,17 +1151,17 @@ Analytic Database to the Hadoop ecosystem.</p>
-
-
-
-
-
+ <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
+
+
+
+</ul>
+
-
+ <h4>Tools</h4>
- <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
-
+<ul>
@@ -2098,11 +1181,17 @@ Analytic Database to the Hadoop ecosystem.</p>
+ <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
+
+
+
-
+
+
+
@@ -2124,7 +1213,7 @@ Analytic Database to the Hadoop ecosystem.</p>
- <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+ <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/building.html
----------------------------------------------------------------------
diff --git a/docs/building.html b/docs/building.html
index bbe1ec4..378f541 100644
--- a/docs/building.html
+++ b/docs/building.html
@@ -109,12 +109,6 @@
-
-
-
-
-
-
<option value="/docs/index.html">Background</option>
@@ -130,14 +124,6 @@
-
-
-
-
-
-
-
-
@@ -174,20 +160,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -221,20 +193,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
<option value="/docs/types.html">Types</option>
@@ -261,12 +219,6 @@
-
-
-
-
-
-
<option value="/docs/indexes.html">Indexes</option>
@@ -280,14 +232,6 @@
-
-
-
-
-
-
-
-
@@ -324,20 +268,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
</optgroup>
@@ -381,20 +311,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
@@ -426,25 +342,11 @@
-
-
-
-
-
-
<option value="/docs/releases.html">Releases</option>
-
-
-
-
-
-
-
-
</optgroup>
@@ -471,12 +373,6 @@
-
-
-
-
-
-
<option value="/docs/hive-ddl.html">Hive DDL</option>
@@ -494,14 +390,6 @@
-
-
-
-
-
-
-
-
@@ -519,12 +407,6 @@
-
-
-
-
-
-
<option value="/docs/hive-config.html">Hive Configuration</option>
@@ -544,14 +426,6 @@
-
-
-
-
-
-
-
-
</optgroup>
@@ -586,12 +460,6 @@
-
-
-
-
-
-
<option value="/docs/mapred.html">Using in MapRed</option>
@@ -601,14 +469,6 @@
-
-
-
-
-
-
-
-
@@ -638,12 +498,6 @@
-
-
-
-
-
-
<option value="/docs/mapreduce.html">Using in MapReduce</option>
@@ -651,14 +505,6 @@
-
-
-
-
-
-
-
-
</optgroup>
@@ -679,8 +525,6 @@
-
-
<option value="/docs/core-java.html">Using Core Java</option>
@@ -704,18 +548,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -727,8 +559,6 @@
-
-
<option value="/docs/core-cpp.html">Using Core C++</option>
@@ -754,18 +584,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
</optgroup>
@@ -788,8 +606,6 @@
-
-
<option value="/docs/cpp-tools.html">C++ Tools</option>
@@ -811,18 +627,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
@@ -848,12 +652,6 @@
-
-
-
-
-
-
<option value="/docs/java-tools.html">Java Tools</option>
@@ -865,383 +663,18 @@
-
-
-
-
-
-
-
-
</optgroup>
- <optgroup label="Format Specification">
-
+ </select>
+</div>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/file-tail.html">File Tail</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/compression.html">Compression</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/stripes.html">Stripes</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/encodings.html">Column Encodings</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-index.html">Indexes</option>
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Building ORC</h1>
- <h2 id="building-both-c-and-java">Building both C++ and Java</h2>
+ <div class="unit four-fifths">
+ <article>
+ <h1>Building ORC</h1>
+ <h2 id="building-both-c-and-java">Building both C++ and Java</h2>
<p>The C++ library is supported on the following operating systems:</p>
@@ -1338,276 +771,50 @@ is invoking:</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/acid.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/releases.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
-
-
- </article>
- </div>
-
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
-
- <h4>Installing</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class="current"><a href="/docs/building.html">Building ORC</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
-
-
+
-
-
+
-
-
+
-
-
+
-
-
+
-
+ <div class="section-nav">
+ <div class="left align-right">
+
+
+
+ <a href="/docs/acid.html" class="prev">Back</a>
+
+ </div>
+ <div class="right align-left">
+
+
+
+ <a href="/docs/releases.html" class="next">Next</a>
+
+ </div>
+ </div>
+ <div class="clear"></div>
- <li class=""><a href="/docs/releases.html">Releases</a></li>
-
+ </article>
+ </div>
-</ul>
-
+ <div class="unit one-fifth hide-on-mobiles">
+ <aside>
- <h4>Using in Hive</h4>
+ <h4>Overview</h4>
<ul>
@@ -1636,11 +843,7 @@ is invoking:</p>
-
-
-
-
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
+ <li class=""><a href="/docs/index.html">Background</a></li>
@@ -1654,34 +857,10 @@ is invoking:</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
+ <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
-</ul>
-
-
- <h4>Using in MapReduce</h4>
-
-
-<ul>
-
@@ -1718,7 +897,7 @@ is invoking:</p>
- <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
+ <li class=""><a href="/docs/types.html">Types</a></li>
@@ -1748,49 +927,7 @@ is invoking:</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
-
-
-
-</ul>
-
-
- <h4>Using ORC Core</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
+ <li class=""><a href="/docs/indexes.html">Indexes</a></li>
@@ -1802,22 +939,14 @@ is invoking:</p>
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
+ <li class=""><a href="/docs/acid.html">ACID support</a></li>
</ul>
- <h4>Tools</h4>
+ <h4>Installing</h4>
<ul>
@@ -1834,15 +963,7 @@ is invoking:</p>
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
+ <li class="current"><a href="/docs/building.html">Building ORC</a></li>
@@ -1880,14 +1001,14 @@ is invoking:</p>
- <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
+ <li class=""><a href="/docs/releases.html">Releases</a></li>
</ul>
- <h4>Format Specification</h4>
+ <h4>Using in Hive</h4>
<ul>
@@ -1914,31 +1035,7 @@ is invoking:</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
+ <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
@@ -1962,31 +1059,17 @@ is invoking:</p>
-
-
-
-
- <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
+ <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-
-
+</ul>
-
-
-
-
-
-
+ <h4>Using in MapReduce</h4>
- <li class=""><a href="/docs/compression.html">Compression</a></li>
-
+<ul>
@@ -2018,19 +1101,7 @@ is invoking:</p>
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
+ <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
@@ -2066,13 +1137,25 @@ is invoking:</p>
-
+ <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
+
+
+
+</ul>
+
-
+ <h4>Using ORC Core</h4>
+
+<ul>
+
+
+
+
+
@@ -2082,7 +1165,7 @@ is invoking:</p>
- <li class=""><a href="/docs/stripes.html">Stripes</a></li>
+ <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
@@ -2100,17 +1183,17 @@ is invoking:</p>
-
-
-
-
-
+ <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
+
+
+
+</ul>
+
-
+ <h4>Tools</h4>
- <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
-
+<ul>
@@ -2130,11 +1213,17 @@ is invoking:</p>
+ <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
+
+
+
-
+
+
+
@@ -2156,7 +1245,7 @@ is invoking:</p>
- <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
+ <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
http://git-wip-us.apache.org/repos/asf/orc/blob/c6e29090/docs/compression.html
----------------------------------------------------------------------
diff --git a/docs/compression.html b/docs/compression.html
deleted file mode 100644
index 2c70cb8..0000000
--- a/docs/compression.html
+++ /dev/null
@@ -1,2193 +0,0 @@
-<!DOCTYPE HTML>
-<html lang="en-US">
-<head>
- <meta charset="UTF-8">
- <title>Compression</title>
- <meta name="viewport" content="width=device-width,initial-scale=1">
- <meta name="generator" content="Jekyll v2.4.0">
- <link rel="stylesheet" href="//fonts.googleapis.com/css?family=Lato:300,300italic,400,400italic,700,700italic,900">
- <link rel="stylesheet" href="/css/screen.css">
- <link rel="icon" type="image/x-icon" href="/favicon.ico">
- <!--[if lt IE 9]>
- <script src="/js/html5shiv.min.js"></script>
- <script src="/js/respond.min.js"></script>
- <![endif]-->
-</head>
-
-
-<body class="wrap">
- <header role="banner">
- <nav class="mobile-nav show-on-mobiles">
- <ul>
- <li class="">
- <a href="/">Home</a>
- </li>
- <li class="current">
- <a href="/docs/"><span class="show-on-mobiles">Docs</span>
- <span class="hide-on-mobiles">Documentation</span></a>
- </li>
- <li class="">
- <a href="/talks/">Talks</a>
- </li>
- <li class="">
- <a href="/news/">News</a>
- </li>
- <li class="">
- <a href="/help/">Help</a>
- </li>
- <li class="">
- <a href="/develop/">Develop</a>
- </li>
-</ul>
-
- </nav>
- <div class="grid">
- <div class="unit one-third center-on-mobiles">
- <h1>
- <a href="/">
- <span class="sr-only">Apache ORC</span>
- <img src="/img/logo.png" width="249" height="101" alt="ORC Logo">
- </a>
- </h1>
- </div>
- <nav class="main-nav unit two-thirds hide-on-mobiles">
- <ul>
- <li class="">
- <a href="/">Home</a>
- </li>
- <li class="current">
- <a href="/docs/"><span class="show-on-mobiles">Docs</span>
- <span class="hide-on-mobiles">Documentation</span></a>
- </li>
- <li class="">
- <a href="/talks/">Talks</a>
- </li>
- <li class="">
- <a href="/news/">News</a>
- </li>
- <li class="">
- <a href="/help/">Help</a>
- </li>
- <li class="">
- <a href="/develop/">Develop</a>
- </li>
-</ul>
-
- </nav>
- </div>
-</header>
-
-
- <section class="docs">
- <div class="grid">
-
- <div class="docs-nav-mobile unit whole show-on-mobiles">
- <select onchange="if (this.value) window.location.href=this.value">
- <option value="">Navigate the docs…</option>
-
- <optgroup label="Overview">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/index.html">Background</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/adopters.html">ORC Adopters</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/types.html">Types</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/indexes.html">Indexes</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/acid.html">ACID support</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- <optgroup label="Installing">
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/building.html">Building ORC</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/releases.html">Releases</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- <optgroup label="Using in Hive">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/hive-ddl.html">Hive DDL</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/hive-config.html">Hive Configuration</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- <optgroup label="Using in MapReduce">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/mapred.html">Using in MapRed</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/mapreduce.html">Using in MapReduce</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- <optgroup label="Using ORC Core">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/core-java.html">Using Core Java</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/core-cpp.html">Using Core C++</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- <optgroup label="Tools">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/cpp-tools.html">C++ Tools</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/java-tools.html">Java Tools</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- <optgroup label="Format Specification">
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-intro.html">Introduction</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/file-tail.html">File Tail</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/compression.html">Compression</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/run-length.html">Run Length Encoding</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/stripes.html">Stripes</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/encodings.html">Column Encodings</option>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <option value="/docs/spec-index.html">Indexes</option>
-
-
-
-
-
-
-
-
-
-
- </optgroup>
-
- </select>
-</div>
-
-
- <div class="unit four-fifths">
- <article>
- <h1>Compression</h1>
- <p>If the ORC file writer selects a generic compression codec (zlib or
-snappy), every part of the ORC file except for the Postscript is
-compressed with that codec. However, one of the requirements for ORC
-is that the reader be able to skip over compressed bytes without
-decompressing the entire stream. To manage this, ORC writes compressed
-streams in chunks with headers as in the figure below.
-To handle uncompressable data, if the compressed data is larger than
-the original, the original is stored and the isOriginal flag is
-set. Each header is 3 bytes long with (compressedLength * 2 +
-isOriginal) stored as a little endian value. For example, the header
-for a chunk that compressed to 100,000 bytes would be [0x40, 0x0d,
-0x03]. The header for 5 bytes that did not compress would be [0x0b,
-0x00, 0x00]. Each compression chunk is compressed independently so
-that as long as a decompressor starts at the top of a header, it can
-start decompressing without the previous bytes.</p>
-
-<p><img src="/img/CompressionStream.png" alt="compression streams" /></p>
-
-<p>The default compression chunk size is 256K, but writers can choose
-their own value. Larger chunks lead to better compression, but require
-more memory. The chunk size is recorded in the Postscript so that
-readers can allocate appropriately sized buffers. Readers are
-guaranteed that no chunk will expand to more than the compression chunk
-size.</p>
-
-<p>ORC files without generic compression write each stream directly
-with no headers.</p>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <div class="section-nav">
- <div class="left align-right">
-
-
-
- <a href="/docs/file-tail.html" class="prev">Back</a>
-
- </div>
- <div class="right align-left">
-
-
-
- <a href="/docs/run-length.html" class="next">Next</a>
-
- </div>
- </div>
- <div class="clear"></div>
-
-
- </article>
- </div>
-
- <div class="unit one-fifth hide-on-mobiles">
- <aside>
-
- <h4>Overview</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/index.html">Background</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/adopters.html">ORC Adopters</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/types.html">Types</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/indexes.html">Indexes</a></li>
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/acid.html">ACID support</a></li>
-
-
-
-</ul>
-
-
- <h4>Installing</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/building.html">Building ORC</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/releases.html">Releases</a></li>
-
-
-
-</ul>
-
-
- <h4>Using in Hive</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/hive-ddl.html">Hive DDL</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/hive-config.html">Hive Configuration</a></li>
-
-
-
-</ul>
-
-
- <h4>Using in MapReduce</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/mapred.html">Using in MapRed</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/mapreduce.html">Using in MapReduce</a></li>
-
-
-
-</ul>
-
-
- <h4>Using ORC Core</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-java.html">Using Core Java</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/core-cpp.html">Using Core C++</a></li>
-
-
-
-</ul>
-
-
- <h4>Tools</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/cpp-tools.html">C++ Tools</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/java-tools.html">Java Tools</a></li>
-
-
-
-</ul>
-
-
- <h4>Format Specification</h4>
-
-
-<ul>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/spec-intro.html">Introduction</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/file-tail.html">File Tail</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class="current"><a href="/docs/compression.html">Compression</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/run-length.html">Run Length Encoding</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/stripes.html">Stripes</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/encodings.html">Column Encodings</a></li>
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- <li class=""><a href="/docs/spec-index.html">Indexes</a></li>
-
-
-
-</ul>
-
-
- </aside>
-</div>
-
-
- <div class="clear"></div>
-
- </div>
- </section>
-
-
- <footer role="contentinfo">
- <p>The contents of this website are © 2018
- <a href="https://www.apache.org/">Apache Software Foundation</a>
- under the terms of the <a
- href="https://www.apache.org/licenses/LICENSE-2.0.html">
- Apache License v2</a>. Apache ORC and its logo are trademarks
- of the Apache Software Foundation.</p>
-</footer>
-
- <script>
- var anchorForId = function (id) {
- var anchor = document.createElement("a");
- anchor.className = "header-link";
- anchor.href = "#" + id;
- anchor.innerHTML = "<span class=\"sr-only\">Permalink</span><i class=\"fa fa-link\"></i>";
- anchor.title = "Permalink";
- return anchor;
- };
-
- var linkifyAnchors = function (level, containingElement) {
- var headers = containingElement.getElementsByTagName("h" + level);
- for (var h = 0; h < headers.length; h++) {
- var header = headers[h];
-
- if (typeof header.id !== "undefined" && header.id !== "") {
- header.appendChild(anchorForId(header.id));
- }
- }
- };
-
- document.onreadystatechange = function () {
- if (this.readyState === "complete") {
- var contentBlock = document.getElementsByClassName("docs")[0] || document.getElementsByClassName("news")[0];
- if (!contentBlock) {
- return;
- }
- for (var level = 1; level <= 6; level++) {
- linkifyAnchors(level, contentBlock);
- }
- }
- };
-</script>
-
-
-</body>
-</html>