You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by gi...@apache.org on 2024/03/06 01:17:56 UTC
(parquet-site) branch asf-site updated: deploy: 5cf926d9c9af6d61e8d24d39e0692d9c7e326066
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/parquet-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new bba117a deploy: 5cf926d9c9af6d61e8d24d39e0692d9c7e326066
bba117a is described below
commit bba117a6e3cc2016ee423e54ab0a6fc2356cffcc
Author: wgtmac <wg...@users.noreply.github.com>
AuthorDate: Wed Mar 6 01:17:50 2024 +0000
deploy: 5cf926d9c9af6d61e8d24d39e0692d9c7e326066
---
output/404.html | 2 +-
output/blog/2021/10/06/1.12.2/index.html | 2 +-
output/blog/2021/10/06/2.9.0/index.html | 2 +-
output/blog/2022/05/26/1.12.3/index.html | 2 +-
output/blog/2023/04/06/1.13.0/index.html | 2 +-
output/blog/2023/05/18/1.13.1/index.html | 2 +-
output/blog/2023/11/20/2.10.0/index.html | 2 +-
output/blog/_print/index.html | 2 +-
output/blog/index.html | 2 +-
output/blog/parquet-format/_print/index.html | 2 +-
output/blog/parquet-format/index.html | 2 +-
output/blog/parquet-mr/_print/index.html | 2 +-
output/blog/parquet-mr/index.html | 2 +-
output/categories/index.html | 2 +-
output/community/_print/index.html | 2 +-
output/community/index.html | 2 +-
output/doap_Parquet.rdf | 6 ++--
output/docs/_print/index.html | 34 +++++++++++-----------
output/docs/asf/_print/index.html | 2 +-
output/docs/asf/donate/index.html | 2 +-
output/docs/asf/events/index.html | 2 +-
output/docs/asf/index.html | 2 +-
output/docs/asf/license/index.html | 2 +-
output/docs/asf/security/index.html | 2 +-
output/docs/asf/sponsor/index.html | 2 +-
output/docs/concepts/_print/index.html | 2 +-
output/docs/concepts/index.html | 2 +-
.../docs/contribution-guidelines/_print/index.html | 2 +-
.../contribution-guidelines/building/index.html | 2 +-
.../contributing/index.html | 2 +-
output/docs/contribution-guidelines/index.html | 2 +-
.../contribution-guidelines/modules/index.html | 2 +-
.../contribution-guidelines/releasing/index.html | 2 +-
output/docs/file-format/_print/index.html | 34 +++++++++++-----------
output/docs/file-format/bloomfilter/index.html | 6 ++--
output/docs/file-format/configurations/index.html | 2 +-
.../docs/file-format/data-pages/_print/index.html | 24 +++++++--------
.../file-format/data-pages/checksumming/index.html | 2 +-
.../file-format/data-pages/columnchunks/index.html | 2 +-
.../file-format/data-pages/compression/index.html | 2 +-
.../file-format/data-pages/encodings/index.html | 20 ++++++-------
.../file-format/data-pages/encryption/index.html | 6 ++--
.../data-pages/errorrecovery/index.html | 2 +-
output/docs/file-format/data-pages/index.html | 2 +-
output/docs/file-format/data-pages/index.xml | 22 +++++++-------
output/docs/file-format/extensibility/index.html | 2 +-
output/docs/file-format/index.html | 4 +--
output/docs/file-format/index.xml | 8 ++---
output/docs/file-format/metadata/index.html | 4 +--
output/docs/file-format/nestedencoding/index.html | 2 +-
output/docs/file-format/nulls/index.html | 2 +-
output/docs/file-format/pageindex/index.html | 4 +--
output/docs/file-format/types/_print/index.html | 2 +-
output/docs/file-format/types/index.html | 2 +-
.../docs/file-format/types/logicaltypes/index.html | 2 +-
output/docs/index.html | 2 +-
output/docs/index.xml | 30 +++++++++----------
output/docs/learning-resources/_print/index.html | 2 +-
.../blog-posts/_print/index.html | 2 +-
.../docs/learning-resources/blog-posts/index.html | 2 +-
output/docs/learning-resources/index.html | 2 +-
.../presentations/_print/index.html | 2 +-
.../conf-2014-parquet-summit-twitter/index.html | 2 +-
.../presentations/hadoop-summit-2014/index.html | 2 +-
.../learning-resources/presentations/index.html | 2 +-
.../presentations/spark-summit-2020/index.html | 2 +-
.../presentations/strata-2013/index.html | 2 +-
output/docs/overview/_print/index.html | 2 +-
output/docs/overview/index.html | 2 +-
output/docs/overview/motivation/index.html | 2 +-
output/index.html | 2 +-
output/tags/index.html | 2 +-
72 files changed, 160 insertions(+), 160 deletions(-)
diff --git a/output/404.html b/output/404.html
index 82db13e..03c0dce 100644
--- a/output/404.html
+++ b/output/404.html
@@ -1,3 +1,3 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content><meta property="og:title" content="404 Page not found"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/404.html"><meta itemprop=name content="404 Page not found"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="404 Page not found"><meta name=twitter:description content="T [...]
<small class=ml-1><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></small></div></div></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script><script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFh [...]
\ No newline at end of file
diff --git a/output/blog/2021/10/06/1.12.2/index.html b/output/blog/2021/10/06/1.12.2/index.html
index 7ee3045..59a482b 100644
--- a/output/blog/2021/10/06/1.12.2/index.html
+++ b/output/blog/2021/10/06/1.12.2/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The latest version of parquet-mr is 1.12.2.
To check the validity of this release, use its:
diff --git a/output/blog/2021/10/06/2.9.0/index.html b/output/blog/2021/10/06/2.9.0/index.html
index 5d3cc17..fe90717 100644
--- a/output/blog/2021/10/06/2.9.0/index.html
+++ b/output/blog/2021/10/06/2.9.0/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The latest version of parquet-format is 2.9.0.
To check the validity of this release, use its:
diff --git a/output/blog/2022/05/26/1.12.3/index.html b/output/blog/2022/05/26/1.12.3/index.html
index 471caf7..d59f944 100644
--- a/output/blog/2022/05/26/1.12.3/index.html
+++ b/output/blog/2022/05/26/1.12.3/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The latest version of parquet-mr is 1.12.3.
To check the validity of this release, use its:
diff --git a/output/blog/2023/04/06/1.13.0/index.html b/output/blog/2023/04/06/1.13.0/index.html
index 4b98a2f..69e8899 100644
--- a/output/blog/2023/04/06/1.13.0/index.html
+++ b/output/blog/2023/04/06/1.13.0/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The latest version of parquet-mr is 1.13.0.
To check the validity of this release, use its:
diff --git a/output/blog/2023/05/18/1.13.1/index.html b/output/blog/2023/05/18/1.13.1/index.html
index 2d92d9e..9f2abdd 100644
--- a/output/blog/2023/05/18/1.13.1/index.html
+++ b/output/blog/2023/05/18/1.13.1/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The latest version of parquet-mr is 1.13.1.
To check the validity of this release, use its:
diff --git a/output/blog/2023/11/20/2.10.0/index.html b/output/blog/2023/11/20/2.10.0/index.html
index 40cb253..47447e1 100644
--- a/output/blog/2023/11/20/2.10.0/index.html
+++ b/output/blog/2023/11/20/2.10.0/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The latest version of parquet-format is 2.10.0.
To check the validity of this release, use its:
diff --git a/output/blog/_print/index.html b/output/blog/_print/index.html
index de0f039..24b83d7 100644
--- a/output/blog/_print/index.html
+++ b/output/blog/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><l [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><l [...]
<meta name=description content><meta property="og:title" content="Release"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/blog/"><meta itemprop=name content="Release"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Release"><meta name=twitter:description content="The Apache Parquet Website"><link rel [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/blog/>Return to the regular view of this page</a>.</p></div><h1 class=title>Release</h1><ul><li><a href=#pg-a9373d016caafc4ca68331dc6fa974ca>Parquet Format Releases</a></li><ul><li><a href=#pg-c9e50ac968526979f7dbe2558d97e2b0>2.10.0</a></li><li><a href=#pg-e942f5058827113b1d4e123b925d385b>2.9.0</a></li></ul><li><a href=#pg-94b71b3e66b7e5fc81746c2eec198021>Parquet-mr Releases</a></li><ul><li><a href=#pg-2db4b017 [...]
...
diff --git a/output/blog/index.html b/output/blog/index.html
index b336a21..bc16189 100644
--- a/output/blog/index.html
+++ b/output/blog/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/blog/_print/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/blog/_print/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180 [...]
<meta name=description content><meta property="og:title" content="Release"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/blog/"><meta itemprop=name content="Release"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Release"><meta name=twitter:description content="The Apache Parquet Website"><link rel [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-blog-li><a href=/blog/ class="align-left pl-0 active td-sidebar-l [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/_index.md class=td-page-meta--edit target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Edit this page</a>
diff --git a/output/blog/parquet-format/_print/index.html b/output/blog/parquet-format/_print/index.html
index 40c213f..f409399 100644
--- a/output/blog/parquet-format/_print/index.html
+++ b/output/blog/parquet-format/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/blog/parquet-format/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ico [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/blog/parquet-format/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ico [...]
<meta name=description content><meta property="og:title" content="Parquet Format Releases"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/blog/parquet-format/"><meta itemprop=name content="Parquet Format Releases"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet Format Releases"><meta name=twi [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/blog/parquet-format/>Return to the regular view of this page</a>.</p></div><h1 class=title>Parquet Format Releases</h1><ul><li><a href=#pg-c9e50ac968526979f7dbe2558d97e2b0>2.10.0</a></li><li><a href=#pg-e942f5058827113b1d4e123b925d385b>2.9.0</a></li></ul><div class=content></div></div><div class=td-content><h1 id=pg-c9e50ac968526979f7dbe2558d97e2b0>2.10.0</h1><div class="td-byline mb-4"><time datetime=2023-11-2 [...]
...
diff --git a/output/blog/parquet-format/index.html b/output/blog/parquet-format/index.html
index 265cf01..51d8a18 100644
--- a/output/blog/parquet-format/index.html
+++ b/output/blog/parquet-format/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/blog/parquet-format/_print/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch- [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/blog/parquet-format/_print/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch- [...]
<meta name=description content><meta property="og:title" content="Parquet Format Releases"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/blog/parquet-format/"><meta itemprop=name content="Parquet Format Releases"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet Format Releases"><meta name=twi [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-blog-li><a href=/blog/ class="align-left pl-0 td-side [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/_index.md class=td-page-meta--edit target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Edit this page</a>
diff --git a/output/blog/parquet-mr/_print/index.html b/output/blog/parquet-mr/_print/index.html
index e7098ca..f1ad291 100644
--- a/output/blog/parquet-mr/_print/index.html
+++ b/output/blog/parquet-mr/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/blog/parquet-mr/><link rel=alternate type=application/rss+xml href=/blog/parquet-mr/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x18 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/blog/parquet-mr/><link rel=alternate type=application/rss+xml href=/blog/parquet-mr/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x18 [...]
<meta name=description content><meta property="og:title" content="Parquet-mr Releases"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/blog/parquet-mr/"><meta itemprop=name content="Parquet-mr Releases"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet-mr Releases"><meta name=twitter:description [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/blog/parquet-mr/>Return to the regular view of this page</a>.</p></div><h1 class=title>Parquet-mr Releases</h1><ul><li><a href=#pg-2db4b0173822f111efe7efdcb576b77c>1.13.1</a></li><li><a href=#pg-bc3c5fe45e7e3e04694a6778965d3dfd>1.13.0</a></li><li><a href=#pg-d64a057026fea182fafbc2dee05d3762>1.12.3</a></li><li><a href=#pg-246a573857c6b7e4c29280efba8e8a61>1.12.2</a></li></ul><div class=content></div></div><div cl [...]
...
diff --git a/output/blog/parquet-mr/index.html b/output/blog/parquet-mr/index.html
index f37baaf..0915c58 100644
--- a/output/blog/parquet-mr/index.html
+++ b/output/blog/parquet-mr/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/blog/parquet-mr/_print/><link rel=alternate type=application/rss+xml href=/blog/parquet-mr/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/blog/parquet-mr/_print/><link rel=alternate type=application/rss+xml href=/blog/parquet-mr/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180 [...]
<meta name=description content><meta property="og:title" content="Parquet-mr Releases"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/blog/parquet-mr/"><meta itemprop=name content="Parquet-mr Releases"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet-mr Releases"><meta name=twitter:description [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-blog-li><a href=/blog/ class="align-left pl-0 td-side [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-mr/_index.md class=td-page-meta--edit target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Edit this page</a>
diff --git a/output/categories/index.html b/output/categories/index.html
index aa45f02..1b99c80 100644
--- a/output/categories/index.html
+++ b/output/categories/index.html
@@ -1,3 +1,3 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=application/rss+xml href=/categories/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/fa [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=application/rss+xml href=/categories/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/fa [...]
<meta name=description content><meta property="og:title" content="Categories"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/categories/"><meta itemprop=name content="Categories"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Categories"><meta name=twitter:description content="The Apache Parquet Web [...]
<small class=ml-1><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></small></div></div></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script><script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFh [...]
\ No newline at end of file
diff --git a/output/community/_print/index.html b/output/community/_print/index.html
index 08bd1e4..acdae9d 100644
--- a/output/community/_print/index.html
+++ b/output/community/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes= [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes= [...]
<meta name=description content><meta property="og:title" content="Community"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/community/"><meta itemprop=name content="Community"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Community"><meta name=twitter:description content="The Apache Parquet Website [...]
Discussion and help from your fellow users. To subscribe please email dev-subscribe@parquet.apache.org. View the archive here: https://lists.apache.org/list.html?dev@parquet.apache.org.</li><li title=Twitter><a target=_blank rel=noopener href=https://twitter.com/ApacheParquet><i class="fab fa-twitter"></i> Twitter:</a>
Follow us on Twitter to get the latest news</li><li title="Stack Overflow"><a target=_blank rel=noopener href=https://stackoverflow.com/questions/tagged/parquet><i class="fab fa-stack-overflow"></i> Stack Overflow:</a>
diff --git a/output/community/index.html b/output/community/index.html
index 17fbbe2..1623aaf 100644
--- a/output/community/index.html
+++ b/output/community/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/community/_print/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png siz [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/community/_print/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png siz [...]
<meta name=description content><meta property="og:title" content="Community"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/community/"><meta itemprop=name content="Community"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Community"><meta name=twitter:description content="The Apache Parquet Website [...]
Discussion and help from your fellow users. To subscribe please email dev-subscribe@parquet.apache.org. View the archive here: https://lists.apache.org/list.html?dev@parquet.apache.org.</li><li title=Twitter><a target=_blank rel=noopener href=https://twitter.com/ApacheParquet><i class="fab fa-twitter"></i> Twitter:</a>
Follow us on Twitter to get the latest news</li><li title="Stack Overflow"><a target=_blank rel=noopener href=https://stackoverflow.com/questions/tagged/parquet><i class="fab fa-stack-overflow"></i> Stack Overflow:</a>
diff --git a/output/doap_Parquet.rdf b/output/doap_Parquet.rdf
index 79b5e31..ca14f2c 100644
--- a/output/doap_Parquet.rdf
+++ b/output/doap_Parquet.rdf
@@ -31,14 +31,14 @@
<description>Apache Parquet is an open source, column-oriented data file format designed for efficient data storage and retrieval. It provides efficient data compression and encoding schemes with enhanced performance to handle complex data in bulk. Parquet is available in multiple languages including Java, C++, and Python.</description>
<bug-database rdf:resource="https://issues.apache.org/jira/browse/PARQUET" />
<mailing-list rdf:resource="https://parquet.apache.org/community/" />
- <download-page rdf:resource="https://parquet.apache.org/downloads/" />
+ <download-page rdf:resource="https://parquet.apache.org/blog/2023/05/18/1.13.1/" />
<programming-language>Java</programming-language>
<category rdf:resource="http://projects.apache.org/category/big-data" />
<release>
<Version>
<name>Parquet MR</name>
- <created>2015-04-15</created>
- <revision>1.6.0</revision>
+ <created>2023-05-18</created>
+ <revision>1.13.1</revision>
</Version>
</release>
<repository>
diff --git a/output/docs/_print/index.html b/output/docs/_print/index.html
index 44a7bb7..6cf7fb1 100644
--- a/output/docs/_print/index.html
+++ b/output/docs/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/><link rel=alternate type=application/rss+xml href=/docs/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><l [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/><link rel=alternate type=application/rss+xml href=/docs/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><l [...]
<meta name=description content="Welcome to the documentation for Apache Parquet. Here, you can find information about the Parquet File Format, including specifications and developer …"><meta property="og:title" content="Documentation"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/docs/"><meta itemprop=name content="Documentation"><meta itemprop=description content="The Apache Parquet Websi [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/>Return to the regular view of this page</a>.</p></div><h1 class=title>Documentation</h1><ul><li>1: <a href=#pg-6e17e09fffc1050f46600282def85180>Overview</a></li><ul><li>1.1: <a href=#pg-2e5324b574579d3eb78ce816487d9e18>Motivation</a></li></ul><li>2: <a href=#pg-fcf6186c3c17ed7f2e8577136109a520>Concepts</a></li><ul></ul><li>3: <a href=#pg-52ee54aeff1ffc82031ec74e9a626eba>File Format</a></li><ul><li>3.1: <a [...]
unchanged for describing this file format. The file format is
@@ -33,7 +33,7 @@ start locations. More details on what is contained in the metadata can be found
in the Thrift definition.</p><p>Metadata is written after the data to allow for single pass writing.</p><p>Readers are expected to first read the file metadata to find all the column
chunks they are interested in. The columns chunks should then be read sequentially.</p><p>The format is explicitly designed to separate the metadata from the data. This
allows splitting columns into multiple files, as well as having a single metadata
-file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>3.1 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
+file reference multiple parquet files.</p><p><img alt="File Layout" src=/images/FileLayout.gif></p></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>3.1 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
possible to do larger sequential IO. Larger groups also require more buffering in
the write path (or a two pass write). We recommend large row groups (512MB - 1GB).
Since an entire row group might need to be read, we want it to completely fit on
@@ -44,7 +44,7 @@ allow for more fine grained reading (e.g. single row lookup). Larger page sizes
incur less space overhead (less page headers) and potentially less parsing overhead
(processing headers). Note: for sequential scans, it is not expected to read a page
at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-6762d78210357c1df172dfddcc6fd307>3.2 - Extensibility</h1><p>There are many places in the format for compatible extensions:</p><ul><li>File Version: The file metadata contains a version.</li><li>Encodings: Encodings are specified by enum and more can be added in the future.</li><li>Page types: Additional page types can be added and safely skipped.</ [...]
-header metadata. All thrift structures are serialized using the TCompactProtocol.</p><p><img src=/images/FileFormat.gif alt="File Layout"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>3.4 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
+header metadata. All thrift structures are serialized using the TCompactProtocol.</p><p><img alt="File Layout" src=/images/FileFormat.gif></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>3.4 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
32-bit ints with an efficient encoding. This reduces the complexity of implementing
@@ -278,8 +278,8 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src=/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
-<img src=/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
+<img alt="File Layout - Bloom filter footer" src=/images/FileLayoutBloomFilter2.png></p><p>Or it can be stored between row groups, the file layout looks like:
+<img alt="File Layout - Bloom filter footer" src=/images/FileLayoutBloomFilter1.png></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
data should be encrypted with the column key, and the Bloom filter of other (not sensitive) columns
do not need to be encrypted.</p><p>Bloom filters have two serializable modules - the PageHeader thrift structure (with its internal
@@ -331,13 +331,13 @@ is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=
If any ambiguity arises when implementing this format, the implementation
provided by the <a href=http://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>3.7.2 - Encodings</h1><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supported for types. It is
intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
-stores the data in the following format:</p><ul><li>BOOLEAN: <a href=#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes contained in the array</li></ul><p>For [...]
+stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
dictionary will be stored in a dictionary page per column chunk. The values are stored as integers
-using the <a href=#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
+using the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
or number of distinct values, the encoding will fall back to the plain encoding. The dictionary page is
-written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
+written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=/docs/file-format/data-pages/encodings/#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
followed by the values encoded using RLE/Bit packed described above (with the given bit width).</p><p>Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. Prefer using RLE_DICTIONARY
in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.</p><p><a name=RLE></a></p><h3 id=run-length-encoding--bit-packing-hybrid-rle--3>Run Length Encoding / Bit-Packing Hybrid (RLE = 3)</h3><p>This encoding uses a combination of bit-packing and run length encoding to more efficiently store repeated values.</p><p>The grammar for this encoding looks like this, given a fixed bit-width known in advance:</p><pre tabindex=0><code>rle-bit-packed-hybrid: <length> <encoded [...]
// length is not always prepended, please check the table below for more detail
@@ -354,7 +354,7 @@ rle-run := <rle-header> <repeated-value>
rle-header := varint-encode( (rle-run-len) << 1)
rle-run-len := *see 3 below*
repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
-</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=#BITPACKED>deprecated bit-packing</a> encoding.
+</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
@@ -366,7 +366,7 @@ bit label: HIDEFABC RMNOJKLG VWXSTUPQ
when deserializing more than one byte at at time. This is because 4 bytes can be read into a
32 bit register (or 8 bytes into a 64 bit register) and values can be unpacked just by
shifting and ORing with a mask. (to make this optimization work on a big-endian machine,
-you would have to use the ordering used in the <a href=#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
+you would have to use the ordering used in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
This means that a Parquet implementation can always store the run length in a signed
32-bit integer. This length restriction was not part of the Parquet 2.5.0 and earlier
specifications, but longer runs were not readable by the most common Parquet
@@ -384,13 +384,13 @@ data:</p><ul><li>Repetition and definition levels</li><li>Dictionary indices</li
| | Dictionary indices | N |
| | Boolean values | Y |
+--------------+------------------------+-----------------+
-</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=#RLE>RLE/bit-packing</a> hybrid encoding.
+</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid encoding.
Each value is encoded back to back using a fixed width.
There is no padding between values (except for the last byte, which is padded with 0s).
For example, if the max repetition level was 3 (2 bits) and the max definition level as 3
-(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
+(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
For compatibility reasons, this implementation packs values from the most significant bit to the least significant bit,
-which is not the same as the <a href=#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+which is not the same as the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 00000101 00111001 01110111
@@ -436,7 +436,7 @@ but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
8 (block size), 1 (miniblock count), 8 (value count), 7 (first value)</p><p>block:
--2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=#RLE>RLE/bit-packing</a> encoding. However the <a href=#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
+-2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding. However the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for [...]
The delta encoding algorithm described above stores a bit width per miniblock and is less sensitive to variations in the size of encoded integers. It is also somewhat doing RLE encoding as a block containing all the same values will be bit packed to a zero bit width thus being only a header.</p><h3 id=delta-length-byte-array-delta_length_byte_array--6>Delta-length byte array: (DELTA_LENGTH_BYTE_ARRAY = 6)</h3><p>Supported Types: BYTE_ARRAY</p><p>This encoding is always preferred over PLA [...]
encoding (DELTA_BINARY_PACKED). The byte array data follows all of the length data just
concatenated back to back. The expected savings is from the cost of encoding the lengths
@@ -688,7 +688,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionEF.png></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -732,7 +732,7 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionPF.png></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
@@ -788,7 +788,7 @@ column are skipped, the corresponding rows in the other columns have to be
skipped. Hence the OffsetIndexes for each column in a RowGroup are stored
together.</li></ul><p>The new index structures are stored separately from RowGroup, near the footer.<br>This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
-location and length are stored in ColumnChunk.</p><p><img src=/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
+location and length are stored in ColumnChunk.</p><p><img alt="Page Index Layout" src=/images/PageIndexLayout.png></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
bound for the last page, because the row group Statistics can provide that.
We still include those for the sake of uniformity, and the overhead should be
negligible.</li><li>We store lower and upper bounds for the values of each page. These may be the
diff --git a/output/docs/asf/_print/index.html b/output/docs/asf/_print/index.html
index 43ee44d..84af215 100644
--- a/output/docs/asf/_print/index.html
+++ b/output/docs/asf/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=18 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=18 [...]
<meta name=description content="Apache Software Foundation
"><meta property="og:title" content="Apache Software Foundation (ASF)"><meta property="og:description" content="Apache Software Foundation
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/asf/"><meta itemprop=name content="Apache Software Foundation (ASF)"><meta itemprop=description content="Apache Software Foundation
diff --git a/output/docs/asf/donate/index.html b/output/docs/asf/donate/index.html
index 66cd724..8d042b6 100644
--- a/output/docs/asf/donate/index.html
+++ b/output/docs/asf/donate/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Donate
"><meta property="og:title" content="Donate"><meta property="og:description" content="Donate"><meta property="og:type" content="article"><meta property="og:url" content="/docs/asf/donate/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Donate"><meta itemprop=description content="Donate"><meta itemprop=dateModified content="2022-03-24T09:39:00-04:00"><meta itemprop=wordCount content="1 [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/asf/events/index.html b/output/docs/asf/events/index.html
index 6d9a4fc..b47110a 100644
--- a/output/docs/asf/events/index.html
+++ b/output/docs/asf/events/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Events
"><meta property="og:title" content="Events"><meta property="og:description" content="Events"><meta property="og:type" content="article"><meta property="og:url" content="/docs/asf/events/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Events"><meta itemprop=description content="Events"><meta itemprop=dateModified content="2022-03-24T09:39:00-04:00"><meta itemprop=wordCount content="1 [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/asf/index.html b/output/docs/asf/index.html
index 6db58ff..b60d2ea 100644
--- a/output/docs/asf/index.html
+++ b/output/docs/asf/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/asf/_print/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/asf/_print/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes [...]
<meta name=description content="Apache Software Foundation
"><meta property="og:title" content="Apache Software Foundation (ASF)"><meta property="og:description" content="Apache Software Foundation
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/asf/"><meta itemprop=name content="Apache Software Foundation (ASF)"><meta itemprop=description content="Apache Software Foundation
diff --git a/output/docs/asf/license/index.html b/output/docs/asf/license/index.html
index fbe0c00..aa53f61 100644
--- a/output/docs/asf/license/index.html
+++ b/output/docs/asf/license/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="License
"><meta property="og:title" content="License"><meta property="og:description" content="License"><meta property="og:type" content="article"><meta property="og:url" content="/docs/asf/license/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="License"><meta itemprop=description content="License"><meta itemprop=dateModified content="2022-03-24T09:39:00-04:00"><meta itemprop=wordCount conte [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/asf/security/index.html b/output/docs/asf/security/index.html
index add3ab4..4950db4 100644
--- a/output/docs/asf/security/index.html
+++ b/output/docs/asf/security/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Security
"><meta property="og:title" content="Security"><meta property="og:description" content="Security"><meta property="og:type" content="article"><meta property="og:url" content="/docs/asf/security/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Security"><meta itemprop=description content="Security"><meta itemprop=dateModified content="2022-03-24T09:39:00-04:00"><meta itemprop=wordCount [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/asf/sponsor/index.html b/output/docs/asf/sponsor/index.html
index 0362f6c..0895a70 100644
--- a/output/docs/asf/sponsor/index.html
+++ b/output/docs/asf/sponsor/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Sponsor
"><meta property="og:title" content="Sponsor"><meta property="og:description" content="Sponsor"><meta property="og:type" content="article"><meta property="og:url" content="/docs/asf/sponsor/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Sponsor"><meta itemprop=description content="Sponsor"><meta itemprop=dateModified content="2022-03-24T09:39:00-04:00"><meta itemprop=wordCount conte [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/concepts/_print/index.html b/output/docs/concepts/_print/index.html
index 78c8a0d..0312360 100644
--- a/output/docs/concepts/_print/index.html
+++ b/output/docs/concepts/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/concepts/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.pn [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/concepts/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.pn [...]
<meta name=description content="Glossary of relevant terminology.
"><meta property="og:title" content="Concepts"><meta property="og:description" content="Glossary of relevant terminology.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/concepts/"><meta itemprop=name content="Concepts"><meta itemprop=description content="Glossary of relevant terminology.
diff --git a/output/docs/concepts/index.html b/output/docs/concepts/index.html
index 16c3721..3afd10f 100644
--- a/output/docs/concepts/index.html
+++ b/output/docs/concepts/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/concepts/_print/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/concepts/_print/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180 [...]
<meta name=description content="Glossary of relevant terminology.
"><meta property="og:title" content="Concepts"><meta property="og:description" content="Glossary of relevant terminology.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/concepts/"><meta itemprop=name content="Concepts"><meta itemprop=description content="Glossary of relevant terminology.
diff --git a/output/docs/contribution-guidelines/_print/index.html b/output/docs/contribution-guidelines/_print/index.html
index 9520d4c..aee1696 100644
--- a/output/docs/contribution-guidelines/_print/index.html
+++ b/output/docs/contribution-guidelines/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/contribution-guidelines/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favico [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/contribution-guidelines/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favico [...]
<meta name=description content="All developer resources related to Parquet.
"><meta property="og:title" content="Developer Guide"><meta property="og:description" content="All developer resources related to Parquet.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/contribution-guidelines/"><meta itemprop=name content="Developer Guide"><meta itemprop=description content="All developer resources related to Parquet.
diff --git a/output/docs/contribution-guidelines/building/index.html b/output/docs/contribution-guidelines/building/index.html
index 96bf20c..dcd44b5 100644
--- a/output/docs/contribution-guidelines/building/index.html
+++ b/output/docs/contribution-guidelines/building/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="How to build Parquet
"><meta property="og:title" content="Building Parquet"><meta property="og:description" content="How to build Parquet
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/contribution-guidelines/building/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Building Parquet"><meta itemprop=description content="How to build Parquet
diff --git a/output/docs/contribution-guidelines/contributing/index.html b/output/docs/contribution-guidelines/contributing/index.html
index ad9046f..5ddf25d 100644
--- a/output/docs/contribution-guidelines/contributing/index.html
+++ b/output/docs/contribution-guidelines/contributing/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="How to contribute to Parquet
"><meta property="og:title" content="Contributing to Parquet"><meta property="og:description" content="How to contribute to Parquet
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/contribution-guidelines/contributing/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-07-09T16:12:08-07:00"><meta itemprop=name content="Contributing to Parquet"><meta itemprop=description content="How to contribute to Parquet
diff --git a/output/docs/contribution-guidelines/index.html b/output/docs/contribution-guidelines/index.html
index 087ec13..27c9717 100644
--- a/output/docs/contribution-guidelines/index.html
+++ b/output/docs/contribution-guidelines/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/contribution-guidelines/_print/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/fav [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/contribution-guidelines/_print/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/fav [...]
<meta name=description content="All developer resources related to Parquet.
"><meta property="og:title" content="Developer Guide"><meta property="og:description" content="All developer resources related to Parquet.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/contribution-guidelines/"><meta itemprop=name content="Developer Guide"><meta itemprop=description content="All developer resources related to Parquet.
diff --git a/output/docs/contribution-guidelines/modules/index.html b/output/docs/contribution-guidelines/modules/index.html
index 027efeb..a2f2701 100644
--- a/output/docs/contribution-guidelines/modules/index.html
+++ b/output/docs/contribution-guidelines/modules/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="The parquet-format project contains format specifications and Thrift definitions of metadata required to properly read Parquet files.
The parquet-mr …"><meta property="og:title" content="Modules"><meta property="og:description" content="The parquet-format project contains format specifications and Thrift definitions of metadata required to properly read Parquet files.
The parquet-mr project contains multiple sub-modules, which implement the core components of reading and writing a nested, column-oriented data stream, map this core onto the parquet format, and provide Hadoop Input/Output Formats, Pig loaders, and other Java-based utilities for interacting with Parquet.
diff --git a/output/docs/contribution-guidelines/releasing/index.html b/output/docs/contribution-guidelines/releasing/index.html
index e884f4a..e145275 100644
--- a/output/docs/contribution-guidelines/releasing/index.html
+++ b/output/docs/contribution-guidelines/releasing/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="How to release Parquet
"><meta property="og:title" content="Releasing Parquet"><meta property="og:description" content="How to release Parquet
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/contribution-guidelines/releasing/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2023-04-06T10:48:15+08:00"><meta itemprop=name content="Releasing Parquet"><meta itemprop=description content="How to release Parquet
diff --git a/output/docs/file-format/_print/index.html b/output/docs/file-format/_print/index.html
index 00c2e00..0846ff3 100644
--- a/output/docs/file-format/_print/index.html
+++ b/output/docs/file-format/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/file-format/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/file-format/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x [...]
<meta name=description content="Documentation about the Parquet File Format.
"><meta property="og:title" content="File Format"><meta property="og:description" content="Documentation about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/file-format/"><meta itemprop=name content="File Format"><meta itemprop=description content="Documentation about the Parquet File Format.
@@ -27,7 +27,7 @@ start locations. More details on what is contained in the metadata can be found
in the Thrift definition.</p><p>Metadata is written after the data to allow for single pass writing.</p><p>Readers are expected to first read the file metadata to find all the column
chunks they are interested in. The columns chunks should then be read sequentially.</p><p>The format is explicitly designed to separate the metadata from the data. This
allows splitting columns into multiple files, as well as having a single metadata
-file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p></div></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>1 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
+file reference multiple parquet files.</p><p><img alt="File Layout" src=/images/FileLayout.gif></p></div></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>1 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
possible to do larger sequential IO. Larger groups also require more buffering in
the write path (or a two pass write). We recommend large row groups (512MB - 1GB).
Since an entire row group might need to be read, we want it to completely fit on
@@ -38,7 +38,7 @@ allow for more fine grained reading (e.g. single row lookup). Larger page sizes
incur less space overhead (less page headers) and potentially less parsing overhead
(processing headers). Note: for sequential scans, it is not expected to read a page
at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-6762d78210357c1df172dfddcc6fd307>2 - Extensibility</h1><p>There are many places in the format for compatible extensions:</p><ul><li>File Version: The file metadata contains a version.</li><li>Encodings: Encodings are specified by enum and more can be added in the future.</li><li>Page types: Additional page types can be added and safely skipped.</li [...]
-header metadata. All thrift structures are serialized using the TCompactProtocol.</p><p><img src=/images/FileFormat.gif alt="File Layout"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>4 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
+header metadata. All thrift structures are serialized using the TCompactProtocol.</p><p><img alt="File Layout" src=/images/FileFormat.gif></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>4 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
32-bit ints with an efficient encoding. This reduces the complexity of implementing
@@ -272,8 +272,8 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src=/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
-<img src=/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
+<img alt="File Layout - Bloom filter footer" src=/images/FileLayoutBloomFilter2.png></p><p>Or it can be stored between row groups, the file layout looks like:
+<img alt="File Layout - Bloom filter footer" src=/images/FileLayoutBloomFilter1.png></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
data should be encrypted with the column key, and the Bloom filter of other (not sensitive) columns
do not need to be encrypted.</p><p>Bloom filters have two serializable modules - the PageHeader thrift structure (with its internal
@@ -325,13 +325,13 @@ is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=
If any ambiguity arises when implementing this format, the implementation
provided by the <a href=http://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>7.2 - Encodings</h1><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supported for types. It is
intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
-stores the data in the following format:</p><ul><li>BOOLEAN: <a href=#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes contained in the array</li></ul><p>For [...]
+stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
dictionary will be stored in a dictionary page per column chunk. The values are stored as integers
-using the <a href=#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
+using the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
or number of distinct values, the encoding will fall back to the plain encoding. The dictionary page is
-written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
+written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=/docs/file-format/data-pages/encodings/#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
followed by the values encoded using RLE/Bit packed described above (with the given bit width).</p><p>Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. Prefer using RLE_DICTIONARY
in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.</p><p><a name=RLE></a></p><h3 id=run-length-encoding--bit-packing-hybrid-rle--3>Run Length Encoding / Bit-Packing Hybrid (RLE = 3)</h3><p>This encoding uses a combination of bit-packing and run length encoding to more efficiently store repeated values.</p><p>The grammar for this encoding looks like this, given a fixed bit-width known in advance:</p><pre tabindex=0><code>rle-bit-packed-hybrid: <length> <encoded [...]
// length is not always prepended, please check the table below for more detail
@@ -348,7 +348,7 @@ rle-run := <rle-header> <repeated-value>
rle-header := varint-encode( (rle-run-len) << 1)
rle-run-len := *see 3 below*
repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
-</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=#BITPACKED>deprecated bit-packing</a> encoding.
+</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
@@ -360,7 +360,7 @@ bit label: HIDEFABC RMNOJKLG VWXSTUPQ
when deserializing more than one byte at at time. This is because 4 bytes can be read into a
32 bit register (or 8 bytes into a 64 bit register) and values can be unpacked just by
shifting and ORing with a mask. (to make this optimization work on a big-endian machine,
-you would have to use the ordering used in the <a href=#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
+you would have to use the ordering used in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
This means that a Parquet implementation can always store the run length in a signed
32-bit integer. This length restriction was not part of the Parquet 2.5.0 and earlier
specifications, but longer runs were not readable by the most common Parquet
@@ -378,13 +378,13 @@ data:</p><ul><li>Repetition and definition levels</li><li>Dictionary indices</li
| | Dictionary indices | N |
| | Boolean values | Y |
+--------------+------------------------+-----------------+
-</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=#RLE>RLE/bit-packing</a> hybrid encoding.
+</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid encoding.
Each value is encoded back to back using a fixed width.
There is no padding between values (except for the last byte, which is padded with 0s).
For example, if the max repetition level was 3 (2 bits) and the max definition level as 3
-(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
+(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
For compatibility reasons, this implementation packs values from the most significant bit to the least significant bit,
-which is not the same as the <a href=#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+which is not the same as the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 00000101 00111001 01110111
@@ -430,7 +430,7 @@ but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
8 (block size), 1 (miniblock count), 8 (value count), 7 (first value)</p><p>block:
--2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=#RLE>RLE/bit-packing</a> encoding. However the <a href=#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
+-2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding. However the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for [...]
The delta encoding algorithm described above stores a bit width per miniblock and is less sensitive to variations in the size of encoded integers. It is also somewhat doing RLE encoding as a block containing all the same values will be bit packed to a zero bit width thus being only a header.</p><h3 id=delta-length-byte-array-delta_length_byte_array--6>Delta-length byte array: (DELTA_LENGTH_BYTE_ARRAY = 6)</h3><p>Supported Types: BYTE_ARRAY</p><p>This encoding is always preferred over PLA [...]
encoding (DELTA_BINARY_PACKED). The byte array data follows all of the length data just
concatenated back to back. The expected savings is from the cost of encoding the lengths
@@ -682,7 +682,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionEF.png></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -726,7 +726,7 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionPF.png></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
@@ -782,7 +782,7 @@ column are skipped, the corresponding rows in the other columns have to be
skipped. Hence the OffsetIndexes for each column in a RowGroup are stored
together.</li></ul><p>The new index structures are stored separately from RowGroup, near the footer.<br>This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
-location and length are stored in ColumnChunk.</p><p><img src=/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
+location and length are stored in ColumnChunk.</p><p><img alt="Page Index Layout" src=/images/PageIndexLayout.png></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
bound for the last page, because the row group Statistics can provide that.
We still include those for the sake of uniformity, and the overhead should be
negligible.</li><li>We store lower and upper bounds for the values of each page. These may be the
diff --git a/output/docs/file-format/bloomfilter/index.html b/output/docs/file-format/bloomfilter/index.html
index f7675ed..365e5f3 100644
--- a/output/docs/file-format/bloomfilter/index.html
+++ b/output/docs/file-format/bloomfilter/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Problem statement
In their current format, column statistics and dictionaries can be used for predicate
pushdown. Statistics include minimum and …"><meta property="og:title" content="Bloom Filter"><meta property="og:description" content="Problem statement In their current format, column statistics and dictionaries can be used for predicate pushdown. Statistics include minimum and maximum value, which can be used to filter out values not in the range. Dictionaries are more specific, and readers can filter out values that are between min and max but not in the dictionary. However, when there [...]
@@ -216,8 +216,8 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src=/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
-<img src=/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
+<img alt="File Layout - Bloom filter footer" src=/images/FileLayoutBloomFilter2.png></p><p>Or it can be stored between row groups, the file layout looks like:
+<img alt="File Layout - Bloom filter footer" src=/images/FileLayoutBloomFilter1.png></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
data should be encrypted with the column key, and the Bloom filter of other (not sensitive) columns
do not need to be encrypted.</p><p>Bloom filters have two serializable modules - the PageHeader thrift structure (with its internal
diff --git a/output/docs/file-format/configurations/index.html b/output/docs/file-format/configurations/index.html
index 100176d..d046226 100644
--- a/output/docs/file-format/configurations/index.html
+++ b/output/docs/file-format/configurations/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Row Group Size
Larger row groups allow for larger column chunks which makes it
possible to do larger sequential IO. Larger groups also require more …"><meta property="og:title" content="Configurations"><meta property="og:description" content="Row Group Size Larger row groups allow for larger column chunks which makes it possible to do larger sequential IO. Larger groups also require more buffering in the write path (or a two pass write). We recommend large row groups (512MB - 1GB). Since an entire row group might need to be read, we want it to completely fit on on [...]
diff --git a/output/docs/file-format/data-pages/_print/index.html b/output/docs/file-format/data-pages/_print/index.html
index 9c874cc..3384cd5 100644
--- a/output/docs/file-format/data-pages/_print/index.html
+++ b/output/docs/file-format/data-pages/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/file-format/data-pages/><link rel=alternate type=application/rss+xml href=/docs/file-format/data-pages/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/file-format/data-pages/><link rel=alternate type=application/rss+xml href=/docs/file-format/data-pages/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons [...]
<meta name=description content="For data pages, the 3 pieces of information are encoded back to back, after the page
header. No padding is allowed in the data page.
In order we have: …"><meta property="og:title" content="Data Pages"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/docs/file-format/data-pages/"><meta itemprop=name content="Data Pages"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Data Pages"><meta name=twitter:description content="The Apache Parq [...]
@@ -44,13 +44,13 @@ is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=
If any ambiguity arises when implementing this format, the implementation
provided by the <a href=http://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>2 - Encodings</h1><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supported for types. It is
intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
-stores the data in the following format:</p><ul><li>BOOLEAN: <a href=#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes contained in the array</li></ul><p>For [...]
+stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
dictionary will be stored in a dictionary page per column chunk. The values are stored as integers
-using the <a href=#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
+using the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
or number of distinct values, the encoding will fall back to the plain encoding. The dictionary page is
-written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
+written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=/docs/file-format/data-pages/encodings/#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
followed by the values encoded using RLE/Bit packed described above (with the given bit width).</p><p>Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. Prefer using RLE_DICTIONARY
in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.</p><p><a name=RLE></a></p><h3 id=run-length-encoding--bit-packing-hybrid-rle--3>Run Length Encoding / Bit-Packing Hybrid (RLE = 3)</h3><p>This encoding uses a combination of bit-packing and run length encoding to more efficiently store repeated values.</p><p>The grammar for this encoding looks like this, given a fixed bit-width known in advance:</p><pre tabindex=0><code>rle-bit-packed-hybrid: <length> <encoded [...]
// length is not always prepended, please check the table below for more detail
@@ -67,7 +67,7 @@ rle-run := <rle-header> <repeated-value>
rle-header := varint-encode( (rle-run-len) << 1)
rle-run-len := *see 3 below*
repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
-</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=#BITPACKED>deprecated bit-packing</a> encoding.
+</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
@@ -79,7 +79,7 @@ bit label: HIDEFABC RMNOJKLG VWXSTUPQ
when deserializing more than one byte at at time. This is because 4 bytes can be read into a
32 bit register (or 8 bytes into a 64 bit register) and values can be unpacked just by
shifting and ORing with a mask. (to make this optimization work on a big-endian machine,
-you would have to use the ordering used in the <a href=#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
+you would have to use the ordering used in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
This means that a Parquet implementation can always store the run length in a signed
32-bit integer. This length restriction was not part of the Parquet 2.5.0 and earlier
specifications, but longer runs were not readable by the most common Parquet
@@ -97,13 +97,13 @@ data:</p><ul><li>Repetition and definition levels</li><li>Dictionary indices</li
| | Dictionary indices | N |
| | Boolean values | Y |
+--------------+------------------------+-----------------+
-</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=#RLE>RLE/bit-packing</a> hybrid encoding.
+</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid encoding.
Each value is encoded back to back using a fixed width.
There is no padding between values (except for the last byte, which is padded with 0s).
For example, if the max repetition level was 3 (2 bits) and the max definition level as 3
-(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
+(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
For compatibility reasons, this implementation packs values from the most significant bit to the least significant bit,
-which is not the same as the <a href=#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+which is not the same as the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 00000101 00111001 01110111
@@ -149,7 +149,7 @@ but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
8 (block size), 1 (miniblock count), 8 (value count), 7 (first value)</p><p>block:
--2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=#RLE>RLE/bit-packing</a> encoding. However the <a href=#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
+-2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding. However the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for [...]
The delta encoding algorithm described above stores a bit width per miniblock and is less sensitive to variations in the size of encoded integers. It is also somewhat doing RLE encoding as a block containing all the same values will be bit packed to a zero bit width thus being only a header.</p><h3 id=delta-length-byte-array-delta_length_byte_array--6>Delta-length byte array: (DELTA_LENGTH_BYTE_ARRAY = 6)</h3><p>Supported Types: BYTE_ARRAY</p><p>This encoding is always preferred over PLA [...]
encoding (DELTA_BINARY_PACKED). The byte array data follows all of the length data just
concatenated back to back. The expected savings is from the cost of encoding the lengths
@@ -401,7 +401,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionEF.png></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -445,7 +445,7 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionPF.png></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
diff --git a/output/docs/file-format/data-pages/checksumming/index.html b/output/docs/file-format/data-pages/checksumming/index.html
index 6474a3e..0f3543e 100644
--- a/output/docs/file-format/data-pages/checksumming/index.html
+++ b/output/docs/file-format/data-pages/checksumming/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Pages of all kinds can be individually checksummed. This allows disabling of checksums
at the HDFS file level, to better support single row lookups. …"><meta property="og:title" content="Checksumming"><meta property="og:description" content="Pages of all kinds can be individually checksummed. This allows disabling of checksums at the HDFS file level, to better support single row lookups. Checksums are calculated using the standard CRC32 algorithm - as used in e.g. GZip - on the serialized binary representation of a page (not including the page header itself)."><meta proper [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/file-format/data-pages/columnchunks/index.html b/output/docs/file-format/data-pages/columnchunks/index.html
index 3098081..143bb57 100644
--- a/output/docs/file-format/data-pages/columnchunks/index.html
+++ b/output/docs/file-format/data-pages/columnchunks/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Column chunks are composed of pages written back to back. The pages share a common
header and readers can skip over pages they are not interested in. …"><meta property="og:title" content="Column Chunks"><meta property="og:description" content="Column chunks are composed of pages written back to back. The pages share a common header and readers can skip over pages they are not interested in. The data for the page follows the header and can be compressed and/or encoded. The compression and encoding is specified in the page metadata.
A column chunk might be partly or completely dictionary encoded. It means that dictionary indexes are saved in the data pages instead of the actual values."><meta property="og:type" content="article"><meta property="og:url" content="/docs/file-format/data-pages/columnchunks/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2024-01-14T20:32:15+08:00"><meta itemprop=name content="Column Chunks"><meta itemprop=description content="Column chunk [...]
diff --git a/output/docs/file-format/data-pages/compression/index.html b/output/docs/file-format/data-pages/compression/index.html
index 0ed4cb0..c70f6c8 100644
--- a/output/docs/file-format/data-pages/compression/index.html
+++ b/output/docs/file-format/data-pages/compression/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Overview
Parquet allows the data block inside dictionary pages and data pages to
be compressed for better space efficiency. The Parquet format …"><meta property="og:title" content="Compression"><meta property="og:description" content="Overview Parquet allows the data block inside dictionary pages and data pages to be compressed for better space efficiency. The Parquet format supports several compression covering different areas in the compression ratio / processing cost spectrum.
diff --git a/output/docs/file-format/data-pages/encodings/index.html b/output/docs/file-format/data-pages/encodings/index.html
index edf0e9c..a52531b 100644
--- a/output/docs/file-format/data-pages/encodings/index.html
+++ b/output/docs/file-format/data-pages/encodings/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="
Plain: (PLAIN = 0)
Supported Types: all
@@ -19,13 +19,13 @@ BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes littl
<a href="https://github.com/apache/parquet-site/issues/new?title=Encodings" class=td-page-meta--issue target=_blank rel=noopener><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
<a id=print href=/docs/file-format/data-pages/_print/><i class="fa fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#plain-plain--0>Plain: (PLAIN = 0)</a></li><li><a href=#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</a></li><li><a href=#run-length-encoding--bit-packing-hybrid-rle--3>Run Length Encoding / Bit-Packing Hybrid (RLE = 3)</a></ [...]
intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
-stores the data in the following format:</p><ul><li>BOOLEAN: <a href=#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes contained in the array</li></ul><p>For [...]
+stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
dictionary will be stored in a dictionary page per column chunk. The values are stored as integers
-using the <a href=#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
+using the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
or number of distinct values, the encoding will fall back to the plain encoding. The dictionary page is
-written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
+written first, before the data pages of the column chunk.</p><p>Dictionary page format: the entries in the dictionary using the <a href=/docs/file-format/data-pages/encodings/#PLAIN>plain</a> encoding.</p><p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
followed by the values encoded using RLE/Bit packed described above (with the given bit width).</p><p>Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. Prefer using RLE_DICTIONARY
in a data page and PLAIN in a dictionary page for Parquet 2.0+ files.</p><p><a name=RLE></a></p><h3 id=run-length-encoding--bit-packing-hybrid-rle--3>Run Length Encoding / Bit-Packing Hybrid (RLE = 3)</h3><p>This encoding uses a combination of bit-packing and run length encoding to more efficiently store repeated values.</p><p>The grammar for this encoding looks like this, given a fixed bit-width known in advance:</p><pre tabindex=0><code>rle-bit-packed-hybrid: <length> <encoded [...]
// length is not always prepended, please check the table below for more detail
@@ -42,7 +42,7 @@ rle-run := <rle-header> <repeated-value>
rle-header := varint-encode( (rle-run-len) << 1)
rle-run-len := *see 3 below*
repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
-</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=#BITPACKED>deprecated bit-packing</a> encoding.
+</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
@@ -54,7 +54,7 @@ bit label: HIDEFABC RMNOJKLG VWXSTUPQ
when deserializing more than one byte at at time. This is because 4 bytes can be read into a
32 bit register (or 8 bytes into a 64 bit register) and values can be unpacked just by
shifting and ORing with a mask. (to make this optimization work on a big-endian machine,
-you would have to use the ordering used in the <a href=#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
+you would have to use the ordering used in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding)</p></li><li><p>varint-encode() is ULEB-128 encoding, see <a href=https://en.wikipedia.org/wiki/LEB128>https://en.wikipedia.org/wiki/LEB128</a></p></li><li><p>bit-packed-run-len and rle-run-len must be in the range [1, 2<sup>31</sup> - 1].
This means that a Parquet implementation can always store the run length in a signed
32-bit integer. This length restriction was not part of the Parquet 2.5.0 and earlier
specifications, but longer runs were not readable by the most common Parquet
@@ -72,13 +72,13 @@ data:</p><ul><li>Repetition and definition levels</li><li>Dictionary indices</li
| | Dictionary indices | N |
| | Boolean values | Y |
+--------------+------------------------+-----------------+
-</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=#RLE>RLE/bit-packing</a> hybrid encoding.
+</code></pre><p><a name=BITPACKED></a></p><h3 id=bit-packed-deprecated-bit_packed--4>Bit-packed (Deprecated) (BIT_PACKED = 4)</h3><p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid encoding.
Each value is encoded back to back using a fixed width.
There is no padding between values (except for the last byte, which is padded with 0s).
For example, if the max repetition level was 3 (2 bits) and the max definition level as 3
-(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
+(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p><p>This implementation is deprecated because the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid is a superset of this implementation.
For compatibility reasons, this implementation packs values from the most significant bit to the least significant bit,
-which is not the same as the <a href=#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+which is not the same as the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> hybrid.</p><p>For example, the numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 00000101 00111001 01110111
@@ -124,7 +124,7 @@ but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
8 (block size), 1 (miniblock count), 8 (value count), 7 (first value)</p><p>block:
--2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=#RLE>RLE/bit-packing</a> encoding. However the <a href=#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
+-2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p><h4 id=characteristics>Characteristics</h4><p>This encoding is similar to the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding. However the <a href=/docs/file-format/data-pages/encodings/#RLE>RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for [...]
The delta encoding algorithm described above stores a bit width per miniblock and is less sensitive to variations in the size of encoded integers. It is also somewhat doing RLE encoding as a block containing all the same values will be bit packed to a zero bit width thus being only a header.</p><h3 id=delta-length-byte-array-delta_length_byte_array--6>Delta-length byte array: (DELTA_LENGTH_BYTE_ARRAY = 6)</h3><p>Supported Types: BYTE_ARRAY</p><p>This encoding is always preferred over PLA [...]
encoding (DELTA_BINARY_PACKED). The byte array data follows all of the length data just
concatenated back to back. The expected savings is from the cost of encoding the lengths
diff --git a/output/docs/file-format/data-pages/encryption/index.html b/output/docs/file-format/data-pages/encryption/index.html
index ed27a9b..4f2540a 100644
--- a/output/docs/file-format/data-pages/encryption/index.html
+++ b/output/docs/file-format/data-pages/encryption/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Parquet files containing sensitive information can be protected by the modular encryption
mechanism that encrypts and authenticates the file data and …"><meta property="og:title" content="Parquet Modular Encryption"><meta property="og:description" content="Parquet files containing sensitive information can be protected by the modular encryption mechanism that encrypts and authenticates the file data and metadata - while allowing for a regular Parquet functionality (columnar projection, predicate pushdown, encoding and compression).
1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues."><meta property="og:type" content="article"><meta property="og:url" content="/docs/file-format/data-pages/encryption/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2024-01-14T20:32:15+08:00"><meta itemprop= [...]
@@ -240,7 +240,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionEF.png></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -284,7 +284,7 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img alt="File Layout - Encrypted footer" src=/images/FileLayoutEncryptionPF.png></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
diff --git a/output/docs/file-format/data-pages/errorrecovery/index.html b/output/docs/file-format/data-pages/errorrecovery/index.html
index 872262b..1011dc6 100644
--- a/output/docs/file-format/data-pages/errorrecovery/index.html
+++ b/output/docs/file-format/data-pages/errorrecovery/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="If the file metadata is corrupt, the file is lost. If the column metadata is corrupt,
that column chunk is lost (but column chunks for this column in …"><meta property="og:title" content="Error Recovery"><meta property="og:description" content="If the file metadata is corrupt, the file is lost. If the column metadata is corrupt, that column chunk is lost (but column chunks for this column in other row groups are okay). If a page header is corrupt, the remaining pages in that chunk are lost. If the data within a page is corrupt, that page is lost. The file will be more res [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/file-format/data-pages/index.html b/output/docs/file-format/data-pages/index.html
index 02cb46e..824f0c3 100644
--- a/output/docs/file-format/data-pages/index.html
+++ b/output/docs/file-format/data-pages/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/file-format/data-pages/_print/><link rel=alternate type=application/rss+xml href=/docs/file-format/data-pages/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favic [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/file-format/data-pages/_print/><link rel=alternate type=application/rss+xml href=/docs/file-format/data-pages/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favic [...]
<meta name=description content="For data pages, the 3 pieces of information are encoded back to back, after the page
header. No padding is allowed in the data page.
In order we have: …"><meta property="og:title" content="Data Pages"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/docs/file-format/data-pages/"><meta itemprop=name content="Data Pages"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Data Pages"><meta name=twitter:description content="The Apache Parq [...]
diff --git a/output/docs/file-format/data-pages/index.xml b/output/docs/file-format/data-pages/index.xml
index 5cd8c57..12594e4 100644
--- a/output/docs/file-format/data-pages/index.xml
+++ b/output/docs/file-format/data-pages/index.xml
@@ -63,7 +63,7 @@ intended to be the simplest encoding. Values are encoded back to back.</p>
<p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p>
<ul>
-<li>BOOLEAN: <a href="#BITPACKED">Bit Packed</a>, LSB first</li>
+<li>BOOLEAN: <a href="/docs/file-format/data-pages/encodings/#BITPACKED">Bit Packed</a>, LSB first</li>
<li>INT32: 4 bytes little endian</li>
<li>INT64: 8 bytes little endian</li>
<li>INT96: 12 bytes little endian (deprecated)</li>
@@ -79,10 +79,10 @@ endian, followed by the bytes.</p>
<h3 id="dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8">Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3>
<p>The dictionary encoding builds a dictionary of values encountered in a given column. The
dictionary will be stored in a dictionary page per column chunk. The values are stored as integers
-using the <a href="#RLE">RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
+using the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
or number of distinct values, the encoding will fall back to the plain encoding. The dictionary page is
written first, before the data pages of the column chunk.</p>
-<p>Dictionary page format: the entries in the dictionary using the <a href="#PLAIN">plain</a> encoding.</p>
+<p>Dictionary page format: the entries in the dictionary using the <a href="/docs/file-format/data-pages/encodings/#PLAIN">plain</a> encoding.</p>
<p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
followed by the values encoded using RLE/Bit packed described above (with the given bit width).</p>
<p>Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. Prefer using RLE_DICTIONARY
@@ -108,7 +108,7 @@ rle-run-len := *see 3 below*
repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
</code></pre><ol>
<li>
-<p>The bit-packing here is done in a different order than the one in the <a href="#BITPACKED">deprecated bit-packing</a> encoding.
+<p>The bit-packing here is done in a different order than the one in the <a href="/docs/file-format/data-pages/encodings/#BITPACKED">deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
significant. For example, to pack the same values as the example in the deprecated encoding above:</p>
@@ -123,7 +123,7 @@ bit label: HIDEFABC RMNOJKLG VWXSTUPQ
when deserializing more than one byte at at time. This is because 4 bytes can be read into a
32 bit register (or 8 bytes into a 64 bit register) and values can be unpacked just by
shifting and ORing with a mask. (to make this optimization work on a big-endian machine,
-you would have to use the ordering used in the <a href="#BITPACKED">deprecated bit-packing</a> encoding)</p>
+you would have to use the ordering used in the <a href="/docs/file-format/data-pages/encodings/#BITPACKED">deprecated bit-packing</a> encoding)</p>
</li>
<li>
<p>varint-encode() is ULEB-128 encoding, see <a href="https://en.wikipedia.org/wiki/LEB128">https://en.wikipedia.org/wiki/LEB128</a></p>
@@ -159,14 +159,14 @@ data:</p>
+--------------+------------------------+-----------------+
</code></pre><p><a name="BITPACKED"></a></p>
<h3 id="bit-packed-deprecated-bit_packed--4">Bit-packed (Deprecated) (BIT_PACKED = 4)</h3>
-<p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href="#RLE">RLE/bit-packing</a> hybrid encoding.
+<p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> hybrid encoding.
Each value is encoded back to back using a fixed width.
There is no padding between values (except for the last byte, which is padded with 0s).
For example, if the max repetition level was 3 (2 bits) and the max definition level as 3
(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p>
-<p>This implementation is deprecated because the <a href="#RLE">RLE/bit-packing</a> hybrid is a superset of this implementation.
+<p>This implementation is deprecated because the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> hybrid is a superset of this implementation.
For compatibility reasons, this implementation packs values from the most significant bit to the least significant bit,
-which is not the same as the <a href="#RLE">RLE/bit-packing</a> hybrid.</p>
+which is not the same as the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> hybrid.</p>
<p>For example, the numbers 1 through 7 using bit width 3:</p>
<pre tabindex="0"><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
@@ -270,7 +270,7 @@ but in real cases it would be invalid.</p>
<p>block:
-2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p>
<h4 id="characteristics">Characteristics</h4>
-<p>This encoding is similar to the <a href="#RLE">RLE/bit-packing</a> encoding. However the <a href="#RLE">RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
+<p>This encoding is similar to the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> encoding. However the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
The delta encoding algorithm described above stores a bit width per miniblock and is less sensitive to variations in the size of encoded integers. It is also somewhat doing RLE encoding as a block containing all the same values will be bit packed to a zero bit width thus being only a header.</p>
<h3 id="delta-length-byte-array-delta_length_byte_array--6">Delta-length byte array: (DELTA_LENGTH_BYTE_ARRAY = 6)</h3>
<p>Supported Types: BYTE_ARRAY</p>
@@ -767,7 +767,7 @@ bytes) that they can’t parse this file.</p>
</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"> * and (possibly) columns **/</span>
</span></span><span style="display:flex;"><span> <span style="color:#0000cf;font-weight:bold">2</span><span style="color:#ce5c00;font-weight:bold">:</span> <span style="color:#000">optional</span> <span style="color:#000">binary</span> <span style="color:#000">key_metadata</span>
</span></span><span style="display:flex;"><span><span style="color:#000;font-weight:bold">}</span>
-</span></span></code></pre></div><p><img src="/images/FileLayoutEncryptionEF.png" alt="File Layout - Encrypted footer"></p>
+</span></span></code></pre></div><p><img alt="File Layout - Encrypted footer" src="/images/FileLayoutEncryptionEF.png"></p>
<h3 id="55-plaintext-footer-mode">5.5 Plaintext footer mode</h3>
<p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
@@ -827,7 +827,7 @@ that contains the combined length of the footer and its signature. A final magic
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
that do not include encrypted columns.</p>
-<p><img src="/images/FileLayoutEncryptionPF.png" alt="File Layout - Encrypted footer"></p>
+<p><img alt="File Layout - Encrypted footer" src="/images/FileLayoutEncryptionPF.png"></p>
<h2 id="6-encryption-overhead">6. Encryption Overhead</h2>
<p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
diff --git a/output/docs/file-format/extensibility/index.html b/output/docs/file-format/extensibility/index.html
index 2cbd4fd..5091b4b 100644
--- a/output/docs/file-format/extensibility/index.html
+++ b/output/docs/file-format/extensibility/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="There are many places in the format for compatible extensions:
File Version: The file metadata contains a version.
diff --git a/output/docs/file-format/index.html b/output/docs/file-format/index.html
index ab7eeaa..d4b0a1f 100644
--- a/output/docs/file-format/index.html
+++ b/output/docs/file-format/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/file-format/_print/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-1 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/file-format/_print/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-1 [...]
<meta name=description content="Documentation about the Parquet File Format.
"><meta property="og:title" content="File Format"><meta property="og:description" content="Documentation about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/file-format/"><meta itemprop=name content="File Format"><meta itemprop=description content="Documentation about the Parquet File Format.
@@ -31,5 +31,5 @@ start locations. More details on what is contained in the metadata can be found
in the Thrift definition.</p><p>Metadata is written after the data to allow for single pass writing.</p><p>Readers are expected to first read the file metadata to find all the column
chunks they are interested in. The columns chunks should then be read sequentially.</p><p>The format is explicitly designed to separate the metadata from the data. This
allows splitting columns into multiple files, as well as having a single metadata
-file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/configurations/>Configurations</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/extensibility/>Extensibility</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/metadata/>Metadata</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/types/>Types</a>< [...]
+file reference multiple parquet files.</p><p><img alt="File Layout" src=/images/FileLayout.gif></p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/configurations/>Configurations</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/extensibility/>Extensibility</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/metadata/>Metadata</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/types/>Types</a>< [...]
<small class=ml-1><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></small></div></div></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script><script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFh [...]
\ No newline at end of file
diff --git a/output/docs/file-format/index.xml b/output/docs/file-format/index.xml
index e93b5dd..5f8a6fe 100644
--- a/output/docs/file-format/index.xml
+++ b/output/docs/file-format/index.xml
@@ -21,7 +21,7 @@ at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></de
</ul></description></item><item><title>Docs: Metadata</title><link>/docs/file-format/metadata/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/metadata/</guid><description>
<p>There are three types of metadata: file metadata, column (chunk) metadata and page
header metadata. All thrift structures are serialized using the TCompactProtocol.</p>
-<p><img src="/images/FileFormat.gif" alt="File Layout"></p></description></item><item><title>Docs: Types</title><link>/docs/file-format/types/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/</guid><description>
+<p><img alt="File Layout" src="/images/FileFormat.gif"></p></description></item><item><title>Docs: Types</title><link>/docs/file-format/types/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/</guid><description>
<p>The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
@@ -318,9 +318,9 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src="/images/FileLayoutBloomFilter2.png" alt="File Layout - Bloom filter footer"></p>
+<img alt="File Layout - Bloom filter footer" src="/images/FileLayoutBloomFilter2.png"></p>
<p>Or it can be stored between row groups, the file layout looks like:
-<img src="/images/FileLayoutBloomFilter1.png" alt="File Layout - Bloom filter footer"></p>
+<img alt="File Layout - Bloom filter footer" src="/images/FileLayoutBloomFilter1.png"></p>
<h4 id="encryption">Encryption</h4>
<p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
@@ -408,7 +408,7 @@ together.</li>
This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
location and length are stored in ColumnChunk.</p>
-<p><img src="/images/PageIndexLayout.png" alt="Page Index Layout"></p>
+<p><img alt="Page Index Layout" src="/images/PageIndexLayout.png"></p>
<p>Some observations:</p>
<ul>
<li>We don&rsquo;t need to record the lower bound for the first page and the upper
diff --git a/output/docs/file-format/metadata/index.html b/output/docs/file-format/metadata/index.html
index 76526ef..408653e 100644
--- a/output/docs/file-format/metadata/index.html
+++ b/output/docs/file-format/metadata/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="There are three types of metadata: file metadata, column (chunk) metadata and page
header metadata. All thrift structures are serialized using the …"><meta property="og:title" content="Metadata"><meta property="og:description" content="There are three types of metadata: file metadata, column (chunk) metadata and page header metadata. All thrift structures are serialized using the TCompactProtocol."><meta property="og:type" content="article"><meta property="og:url" content="/docs/file-format/metadata/"><meta property="article:section" content="docs"><meta property="art [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
@@ -6,5 +6,5 @@ header metadata. All thrift structures are serialized using the …"><meta prop
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/metadata.md?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+s [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Metadata" class=td-page-meta--issue target=_blank rel=noopener><i class="fab fa-github fa-fw"></i> Create documentation issue</a>
<a id=print href=/docs/file-format/_print/><i class="fa fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 pl-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page><a href=/docs/file-format/metadata/>Metadata</a></li></ol></na [...]
-header metadata. All thrift structures are serialized using the TCompactProtocol.</p><p><img src=/images/FileFormat.gif alt="File Layout"></p><div class="text-muted mt-5 pt-3 border-top">Last modified January 14, 2024: <a href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="bg-dark py-5 row d-print-none"><div class="container-fluid mx-sm-5"><div class=ro [...]
+header metadata. All thrift structures are serialized using the TCompactProtocol.</p><p><img alt="File Layout" src=/images/FileFormat.gif></p><div class="text-muted mt-5 pt-3 border-top">Last modified January 14, 2024: <a href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="bg-dark py-5 row d-print-none"><div class="container-fluid mx-sm-5"><div class=ro [...]
<small class=ml-1><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></small></div></div></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script><script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFh [...]
\ No newline at end of file
diff --git a/output/docs/file-format/nestedencoding/index.html b/output/docs/file-format/nestedencoding/index.html
index eae3e9b..99b04ce 100644
--- a/output/docs/file-format/nestedencoding/index.html
+++ b/output/docs/file-format/nestedencoding/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="To encode nested columns, Parquet uses the Dremel encoding with definition and
repetition levels. Definition levels specify how many optional fields …"><meta property="og:title" content="Nested Encoding"><meta property="og:description" content="To encode nested columns, Parquet uses the Dremel encoding with definition and repetition levels. Definition levels specify how many optional fields in the path for the column are defined. Repetition levels specify at what repeated field in the path has the value repeated. The max definition and repetition levels can be com [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/file-format/nulls/index.html b/output/docs/file-format/nulls/index.html
index a4c9228..f679ed3 100644
--- a/output/docs/file-format/nulls/index.html
+++ b/output/docs/file-format/nulls/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Nullity is encoded in the definition levels (which is run-length encoded). NULL values
are not encoded in the data. For example, in a non-nested …"><meta property="og:title" content="Nulls"><meta property="og:description" content="Nullity is encoded in the definition levels (which is run-length encoded). NULL values are not encoded in the data. For example, in a non-nested schema, a column with 1000 NULLs would be encoded with run-length encoding (0, 1000 times) for the definition levels and nothing else."><meta property="og:type" content="article"><meta property="og:url" [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/file-format/pageindex/index.html b/output/docs/file-format/pageindex/index.html
index 0c8abf3..fbba498 100644
--- a/output/docs/file-format/pageindex/index.html
+++ b/output/docs/file-format/pageindex/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="This document describes the format for column index pages in the Parquet
footer. These pages contain statistics for DataPages and can be used to skip …"><meta property="og:title" content="Page Index"><meta property="og:description" content="This document describes the format for column index pages in the Parquet footer. These pages contain statistics for DataPages and can be used to skip pages when scanning data in ordered and unordered columns.
Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics."><meta property="og:type" content="article"><meta property="og:url" content="/docs/file-format/pageindex/"><meta property="article:section" content="docs"><meta property="article:modified_time" [...]
@@ -32,7 +32,7 @@ column are skipped, the corresponding rows in the other columns have to be
skipped. Hence the OffsetIndexes for each column in a RowGroup are stored
together.</li></ul><p>The new index structures are stored separately from RowGroup, near the footer.<br>This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
-location and length are stored in ColumnChunk.</p><p><img src=/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
+location and length are stored in ColumnChunk.</p><p><img alt="Page Index Layout" src=/images/PageIndexLayout.png></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
bound for the last page, because the row group Statistics can provide that.
We still include those for the sake of uniformity, and the overhead should be
negligible.</li><li>We store lower and upper bounds for the values of each page. These may be the
diff --git a/output/docs/file-format/types/_print/index.html b/output/docs/file-format/types/_print/index.html
index cce7e3b..238c5e8 100644
--- a/output/docs/file-format/types/_print/index.html
+++ b/output/docs/file-format/types/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/file-format/types/><link rel=alternate type=application/rss+xml href=/docs/file-format/types/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-tou [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/file-format/types/><link rel=alternate type=application/rss+xml href=/docs/file-format/types/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-tou [...]
<meta name=description content="The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, …"><meta property="og:title" content="Types"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/docs/file-format/types/"><meta itemprop=name content="Types"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Types"><meta name=twitter:descri [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/types/>Return to the regular view of this page</a>.</p></div><h1 class=title>Types</h1><ul><li>1: <a href=#pg-967cf8be84f5efbfd2e57475707b79b1>Logical Types</a></li></ul><div class=content><p>The types supported by the file format are intended to be as minimal as possible,
diff --git a/output/docs/file-format/types/index.html b/output/docs/file-format/types/index.html
index d3ea3f9..0f2018c 100644
--- a/output/docs/file-format/types/index.html
+++ b/output/docs/file-format/types/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/file-format/types/_print/><link rel=alternate type=application/rss+xml href=/docs/file-format/types/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple- [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/file-format/types/_print/><link rel=alternate type=application/rss+xml href=/docs/file-format/types/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple- [...]
<meta name=description content="The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, …"><meta property="og:title" content="Types"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/docs/file-format/types/"><meta itemprop=name content="Types"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Types"><meta name=twitter:descri [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/file-format/types/logicaltypes/index.html b/output/docs/file-format/types/logicaltypes/index.html
index c3d6063..217308b 100644
--- a/output/docs/file-format/types/logicaltypes/index.html
+++ b/output/docs/file-format/types/logicaltypes/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Logical types are used to extend the types that parquet can be used to store,
by specifying how the primitive types should be interpreted. This keeps …"><meta property="og:title" content="Logical Types"><meta property="og:description" content="Logical types are used to extend the types that parquet can be used to store, by specifying how the primitive types should be interpreted. This keeps the set of primitive types to a minimum and reuses parquet’s efficient encodings. For example, strings are stored as byte arrays (binary) with a UTF8 annotation. These an [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child active-path" id=m-docs-li><a href=/docs/ class="align-left pl-0 td-side [...]
diff --git a/output/docs/index.html b/output/docs/index.html
index 7a24e69..5478f7c 100644
--- a/output/docs/index.html
+++ b/output/docs/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/_print/><link rel=alternate type=application/rss+xml href=/docs/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/_print/><link rel=alternate type=application/rss+xml href=/docs/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180 [...]
<meta name=description content="Welcome to the documentation for Apache Parquet. Here, you can find information about the Parquet File Format, including specifications and developer …"><meta property="og:title" content="Documentation"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/docs/"><meta itemprop=name content="Documentation"><meta itemprop=description content="The Apache Parquet Websi [...]
<button class="btn btn-link td-sidebar__toggle d-md-none p-0 ml-3 fas fa-bars" type=button data-toggle=collapse data-target=#td-section-nav aria-controls=td-docs-nav aria-expanded=false aria-label="Toggle section navigation"></button></form><nav class="collapse td-sidebar-nav" id=td-section-nav><ul class="td-sidebar-nav__section pr-md-3 ul-0"><li class="td-sidebar-nav__section-title td-sidebar-nav__section with-child" id=m-docs-li><a href=/docs/ class="align-left pl-0 active td-sidebar-l [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/_index.md class=td-page-meta--edit target=_blank rel=noopener><i class="fa fa-edit fa-fw"></i> Edit this page</a>
diff --git a/output/docs/index.xml b/output/docs/index.xml
index 872c2c5..31b1346 100644
--- a/output/docs/index.xml
+++ b/output/docs/index.xml
@@ -63,7 +63,7 @@ intended to be the simplest encoding. Values are encoded back to back.</p>
<p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p>
<ul>
-<li>BOOLEAN: <a href="#BITPACKED">Bit Packed</a>, LSB first</li>
+<li>BOOLEAN: <a href="/docs/file-format/data-pages/encodings/#BITPACKED">Bit Packed</a>, LSB first</li>
<li>INT32: 4 bytes little endian</li>
<li>INT64: 8 bytes little endian</li>
<li>INT96: 12 bytes little endian (deprecated)</li>
@@ -79,10 +79,10 @@ endian, followed by the bytes.</p>
<h3 id="dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8">Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3>
<p>The dictionary encoding builds a dictionary of values encountered in a given column. The
dictionary will be stored in a dictionary page per column chunk. The values are stored as integers
-using the <a href="#RLE">RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
+using the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/Bit-Packing Hybrid</a> encoding. If the dictionary grows too big, whether in size
or number of distinct values, the encoding will fall back to the plain encoding. The dictionary page is
written first, before the data pages of the column chunk.</p>
-<p>Dictionary page format: the entries in the dictionary using the <a href="#PLAIN">plain</a> encoding.</p>
+<p>Dictionary page format: the entries in the dictionary using the <a href="/docs/file-format/data-pages/encodings/#PLAIN">plain</a> encoding.</p>
<p>Data page format: the bit width used to encode the entry ids stored as 1 byte (max bit width = 32),
followed by the values encoded using RLE/Bit packed described above (with the given bit width).</p>
<p>Using the PLAIN_DICTIONARY enum value is deprecated in the Parquet 2.0 specification. Prefer using RLE_DICTIONARY
@@ -108,7 +108,7 @@ rle-run-len := *see 3 below*
repeated-value := value that is repeated, using a fixed-width of round-up-to-next-byte(bit-width)
</code></pre><ol>
<li>
-<p>The bit-packing here is done in a different order than the one in the <a href="#BITPACKED">deprecated bit-packing</a> encoding.
+<p>The bit-packing here is done in a different order than the one in the <a href="/docs/file-format/data-pages/encodings/#BITPACKED">deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
significant. For example, to pack the same values as the example in the deprecated encoding above:</p>
@@ -123,7 +123,7 @@ bit label: HIDEFABC RMNOJKLG VWXSTUPQ
when deserializing more than one byte at at time. This is because 4 bytes can be read into a
32 bit register (or 8 bytes into a 64 bit register) and values can be unpacked just by
shifting and ORing with a mask. (to make this optimization work on a big-endian machine,
-you would have to use the ordering used in the <a href="#BITPACKED">deprecated bit-packing</a> encoding)</p>
+you would have to use the ordering used in the <a href="/docs/file-format/data-pages/encodings/#BITPACKED">deprecated bit-packing</a> encoding)</p>
</li>
<li>
<p>varint-encode() is ULEB-128 encoding, see <a href="https://en.wikipedia.org/wiki/LEB128">https://en.wikipedia.org/wiki/LEB128</a></p>
@@ -159,14 +159,14 @@ data:</p>
+--------------+------------------------+-----------------+
</code></pre><p><a name="BITPACKED"></a></p>
<h3 id="bit-packed-deprecated-bit_packed--4">Bit-packed (Deprecated) (BIT_PACKED = 4)</h3>
-<p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href="#RLE">RLE/bit-packing</a> hybrid encoding.
+<p>This is a bit-packed only encoding, which is deprecated and will be replaced by the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> hybrid encoding.
Each value is encoded back to back using a fixed width.
There is no padding between values (except for the last byte, which is padded with 0s).
For example, if the max repetition level was 3 (2 bits) and the max definition level as 3
(2 bits), to encode 30 values, we would have 30 * 2 = 60 bits = 8 bytes.</p>
-<p>This implementation is deprecated because the <a href="#RLE">RLE/bit-packing</a> hybrid is a superset of this implementation.
+<p>This implementation is deprecated because the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> hybrid is a superset of this implementation.
For compatibility reasons, this implementation packs values from the most significant bit to the least significant bit,
-which is not the same as the <a href="#RLE">RLE/bit-packing</a> hybrid.</p>
+which is not the same as the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> hybrid.</p>
<p>For example, the numbers 1 through 7 using bit width 3:</p>
<pre tabindex="0"><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
@@ -270,7 +270,7 @@ but in real cases it would be invalid.</p>
<p>block:
-2 (minimum delta), 2 (bitwidth), 00000011111111b (0,0,0,3,3,3,3 packed on 2 bits)</p>
<h4 id="characteristics">Characteristics</h4>
-<p>This encoding is similar to the <a href="#RLE">RLE/bit-packing</a> encoding. However the <a href="#RLE">RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
+<p>This encoding is similar to the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> encoding. However the <a href="/docs/file-format/data-pages/encodings/#RLE">RLE/bit-packing</a> encoding is specifically used when the range of ints is small over the entire page, as is true of repetition and definition levels. It uses a single bit width for the whole page.
The delta encoding algorithm described above stores a bit width per miniblock and is less sensitive to variations in the size of encoded integers. It is also somewhat doing RLE encoding as a block containing all the same values will be bit packed to a zero bit width thus being only a header.</p>
<h3 id="delta-length-byte-array-delta_length_byte_array--6">Delta-length byte array: (DELTA_LENGTH_BYTE_ARRAY = 6)</h3>
<p>Supported Types: BYTE_ARRAY</p>
@@ -767,7 +767,7 @@ bytes) that they can’t parse this file.</p>
</span></span></span><span style="display:flex;"><span><span style="color:#8f5902;font-style:italic"> * and (possibly) columns **/</span>
</span></span><span style="display:flex;"><span> <span style="color:#0000cf;font-weight:bold">2</span><span style="color:#ce5c00;font-weight:bold">:</span> <span style="color:#000">optional</span> <span style="color:#000">binary</span> <span style="color:#000">key_metadata</span>
</span></span><span style="display:flex;"><span><span style="color:#000;font-weight:bold">}</span>
-</span></span></code></pre></div><p><img src="/images/FileLayoutEncryptionEF.png" alt="File Layout - Encrypted footer"></p>
+</span></span></code></pre></div><p><img alt="File Layout - Encrypted footer" src="/images/FileLayoutEncryptionEF.png"></p>
<h3 id="55-plaintext-footer-mode">5.5 Plaintext footer mode</h3>
<p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
@@ -827,7 +827,7 @@ that contains the combined length of the footer and its signature. A final magic
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
that do not include encrypted columns.</p>
-<p><img src="/images/FileLayoutEncryptionPF.png" alt="File Layout - Encrypted footer"></p>
+<p><img alt="File Layout - Encrypted footer" src="/images/FileLayoutEncryptionPF.png"></p>
<h2 id="6-encryption-overhead">6. Encryption Overhead</h2>
<p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
@@ -1091,7 +1091,7 @@ Annotations are stored as <code>LogicalType</code> fields in the file meta
documented in LogicalTypes.md.</p></description></item><item><title>Docs: Metadata</title><link>/docs/file-format/metadata/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/metadata/</guid><description>
<p>There are three types of metadata: file metadata, column (chunk) metadata and page
header metadata. All thrift structures are serialized using the TCompactProtocol.</p>
-<p><img src="/images/FileFormat.gif" alt="File Layout"></p></description></item><item><title>Docs: Nested Encoding</title><link>/docs/file-format/nestedencoding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/nestedencoding/</guid><description>
+<p><img alt="File Layout" src="/images/FileFormat.gif"></p></description></item><item><title>Docs: Nested Encoding</title><link>/docs/file-format/nestedencoding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/nestedencoding/</guid><description>
<p>To encode nested columns, Parquet uses the Dremel encoding with definition and
repetition levels. Definition levels specify how many optional fields in the
path for the column are defined. Repetition levels specify at what repeated field
@@ -1374,9 +1374,9 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src="/images/FileLayoutBloomFilter2.png" alt="File Layout - Bloom filter footer"></p>
+<img alt="File Layout - Bloom filter footer" src="/images/FileLayoutBloomFilter2.png"></p>
<p>Or it can be stored between row groups, the file layout looks like:
-<img src="/images/FileLayoutBloomFilter1.png" alt="File Layout - Bloom filter footer"></p>
+<img alt="File Layout - Bloom filter footer" src="/images/FileLayoutBloomFilter1.png"></p>
<h4 id="encryption">Encryption</h4>
<p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
@@ -1474,7 +1474,7 @@ together.</li>
This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
location and length are stored in ColumnChunk.</p>
-<p><img src="/images/PageIndexLayout.png" alt="Page Index Layout"></p>
+<p><img alt="Page Index Layout" src="/images/PageIndexLayout.png"></p>
<p>Some observations:</p>
<ul>
<li>We don&rsquo;t need to record the lower bound for the first page and the upper
diff --git a/output/docs/learning-resources/_print/index.html b/output/docs/learning-resources/_print/index.html
index c4e561e..8add864 100644
--- a/output/docs/learning-resources/_print/index.html
+++ b/output/docs/learning-resources/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/learning-resources/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-t [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/learning-resources/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-t [...]
<meta name=description content="Various resources to learn about the Parquet File Format.
"><meta property="og:title" content="Resources"><meta property="og:description" content="Various resources to learn about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/learning-resources/"><meta itemprop=name content="Resources"><meta itemprop=description content="Various resources to learn about the Parquet File Format.
diff --git a/output/docs/learning-resources/blog-posts/_print/index.html b/output/docs/learning-resources/blog-posts/_print/index.html
index 72cf5b3..a361041 100644
--- a/output/docs/learning-resources/blog-posts/_print/index.html
+++ b/output/docs/learning-resources/blog-posts/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/learning-resources/blog-posts/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/learning-resources/blog-posts/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon [...]
<meta name=description content="Blog posts with content about the Parquet File Format.
"><meta property="og:title" content="Blog Posts"><meta property="og:description" content="Blog posts with content about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/learning-resources/blog-posts/"><meta itemprop=name content="Blog Posts"><meta itemprop=description content="Blog posts with content about the Parquet File Format.
diff --git a/output/docs/learning-resources/blog-posts/index.html b/output/docs/learning-resources/blog-posts/index.html
index 4bac213..3d03de9 100644
--- a/output/docs/learning-resources/blog-posts/index.html
+++ b/output/docs/learning-resources/blog-posts/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/learning-resources/blog-posts/_print/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-ic [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/learning-resources/blog-posts/_print/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-ic [...]
<meta name=description content="Blog posts with content about the Parquet File Format.
"><meta property="og:title" content="Blog Posts"><meta property="og:description" content="Blog posts with content about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/learning-resources/blog-posts/"><meta itemprop=name content="Blog Posts"><meta itemprop=description content="Blog posts with content about the Parquet File Format.
diff --git a/output/docs/learning-resources/index.html b/output/docs/learning-resources/index.html
index 39fc5dd..3b2d8b1 100644
--- a/output/docs/learning-resources/index.html
+++ b/output/docs/learning-resources/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/learning-resources/_print/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/appl [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/learning-resources/_print/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/appl [...]
<meta name=description content="Various resources to learn about the Parquet File Format.
"><meta property="og:title" content="Resources"><meta property="og:description" content="Various resources to learn about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/learning-resources/"><meta itemprop=name content="Resources"><meta itemprop=description content="Various resources to learn about the Parquet File Format.
diff --git a/output/docs/learning-resources/presentations/_print/index.html b/output/docs/learning-resources/presentations/_print/index.html
index 9ad8dc3..695d1a3 100644
--- a/output/docs/learning-resources/presentations/_print/index.html
+++ b/output/docs/learning-resources/presentations/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/learning-resources/presentations/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/learning-resources/presentations/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch [...]
<meta name=description content="Presentations with content about the Parquet File Format.
"><meta property="og:title" content="Presentations"><meta property="og:description" content="Presentations with content about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/learning-resources/presentations/"><meta itemprop=name content="Presentations"><meta itemprop=description content="Presentations with content about the Parquet File Format.
diff --git a/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html b/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html
index adb24ef..1343213 100644
--- a/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html
+++ b/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="#CONF 2014: Parquet Format at Twitter
"><meta property="og:title" content="#CONF 2014"><meta property="og:description" content="#CONF 2014: Parquet Format at Twitter
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="#CONF 2014"><meta itemprop=description content="#CONF 2014: Parquet Format at Twitter
diff --git a/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html b/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html
index 9766851..59b4486 100644
--- a/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html
+++ b/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Hadoop Summit 2014: Efficient Data Storage for Analytics with Parquet 2.0
"><meta property="og:title" content="Hadoop Summit 2014"><meta property="og:description" content="Hadoop Summit 2014: Efficient Data Storage for Analytics with Parquet 2.0
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/learning-resources/presentations/hadoop-summit-2014/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Hadoop Summit 2014"><meta itemprop=description content="Hadoop Summit 2014: Efficient Data Storage for Analytics with Parquet 2.0
diff --git a/output/docs/learning-resources/presentations/index.html b/output/docs/learning-resources/presentations/index.html
index 8c2e2c8..9e6e588 100644
--- a/output/docs/learning-resources/presentations/index.html
+++ b/output/docs/learning-resources/presentations/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/learning-resources/presentations/_print/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-to [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/learning-resources/presentations/_print/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-to [...]
<meta name=description content="Presentations with content about the Parquet File Format.
"><meta property="og:title" content="Presentations"><meta property="og:description" content="Presentations with content about the Parquet File Format.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/learning-resources/presentations/"><meta itemprop=name content="Presentations"><meta itemprop=description content="Presentations with content about the Parquet File Format.
diff --git a/output/docs/learning-resources/presentations/spark-summit-2020/index.html b/output/docs/learning-resources/presentations/spark-summit-2020/index.html
index ea33d1a..7524e54 100644
--- a/output/docs/learning-resources/presentations/spark-summit-2020/index.html
+++ b/output/docs/learning-resources/presentations/spark-summit-2020/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Spark Summit 2020: The Apache Spark File Format Ecosystem
"><meta property="og:title" content="Spark Summit 2020"><meta property="og:description" content="Spark Summit 2020: The Apache Spark File Format Ecosystem
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/learning-resources/presentations/spark-summit-2020/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Spark Summit 2020"><meta itemprop=description content="Spark Summit 2020: The Apache Spark File Format Ecosystem
diff --git a/output/docs/learning-resources/presentations/strata-2013/index.html b/output/docs/learning-resources/presentations/strata-2013/index.html
index c6fe617..fe21add 100644
--- a/output/docs/learning-resources/presentations/strata-2013/index.html
+++ b/output/docs/learning-resources/presentations/strata-2013/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="Strata 2013: Parquet: Columnar storage for the people
"><meta property="og:title" content="Strata 2013"><meta property="og:description" content="Strata 2013: Parquet: Columnar storage for the people
"><meta property="og:type" content="article"><meta property="og:url" content="/docs/learning-resources/presentations/strata-2013/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Strata 2013"><meta itemprop=description content="Strata 2013: Parquet: Columnar storage for the people
diff --git a/output/docs/overview/_print/index.html b/output/docs/overview/_print/index.html
index 42d1c59..7475b08 100644
--- a/output/docs/overview/_print/index.html
+++ b/output/docs/overview/_print/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=canonical type=text/html href=/docs/overview/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.pn [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=canonical type=text/html href=/docs/overview/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.pn [...]
<meta name=description content="All about Parquet.
"><meta property="og:title" content="Overview"><meta property="og:description" content="All about Parquet.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/overview/"><meta itemprop=name content="Overview"><meta itemprop=description content="All about Parquet.
diff --git a/output/docs/overview/index.html b/output/docs/overview/index.html
index cea3789..83b65d4 100644
--- a/output/docs/overview/index.html
+++ b/output/docs/overview/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=text/html href=/docs/overview/_print/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=text/html href=/docs/overview/_print/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180 [...]
<meta name=description content="All about Parquet.
"><meta property="og:title" content="Overview"><meta property="og:description" content="All about Parquet.
"><meta property="og:type" content="website"><meta property="og:url" content="/docs/overview/"><meta itemprop=name content="Overview"><meta itemprop=description content="All about Parquet.
diff --git a/output/docs/overview/motivation/index.html b/output/docs/overview/motivation/index.html
index bd4bdc8..8ede70e 100644
--- a/output/docs/overview/motivation/index.html
+++ b/output/docs/overview/motivation/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicons [...]
<meta name=description content="We created Parquet to make the advantages of compressed, efficient columnar data representation available to any project in the Hadoop ecosystem. …"><meta property="og:title" content="Motivation"><meta property="og:description" content="We created Parquet to make the advantages of compressed, efficient columnar data representation available to any project in the Hadoop ecosystem.
Parquet is built from the ground up with complex nested data structures in mind, and uses the record shredding and assembly algorithm described in the Dremel paper. We believe this approach is superior to simple flattening of nested name spaces.
Parquet is built to support very efficient compression and encoding schemes."><meta property="og:type" content="article"><meta property="og:url" content="/docs/overview/motivation/"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2022-03-24T09:39:00-04:00"><meta itemprop=name content="Motivation"><meta itemprop=description content="We created Parquet to make the advantages of compressed, efficient columnar data representation available to a [...]
diff --git a/output/index.html b/output/index.html
index 61ce254..b4118f3 100644
--- a/output/index.html
+++ b/output/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=application/rss+xml href=/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16 [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=application/rss+xml href=/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16 [...]
<meta name=description content="
diff --git a/output/tags/index.html b/output/tags/index.html
index ec13e80..2e8c2a3 100644
--- a/output/tags/index.html
+++ b/output/tags/index.html
@@ -1,3 +1,3 @@
-<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.121.2"><link rel=alternate type=application/rss+xml href=/tags/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon- [...]
+<!doctype html><html lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=generator content="Hugo 0.123.7"><link rel=alternate type=application/rss+xml href=/tags/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon- [...]
<meta name=description content><meta property="og:title" content="Tags"><meta property="og:description" content="The Apache Parquet Website"><meta property="og:type" content="website"><meta property="og:url" content="/tags/"><meta itemprop=name content="Tags"><meta itemprop=description content="The Apache Parquet Website"><meta name=twitter:card content="summary"><meta name=twitter:title content="Tags"><meta name=twitter:description content="The Apache Parquet Website"><link rel=preload [...]
<small class=ml-1><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></small></div></div></div></footer></div><script src=https://cdn.jsdelivr.net/npm/popper.js@1.16.1/dist/umd/popper.min.js integrity=sha384-9/reFTGAW83EW2RDu2S0VKaIzap3H66lZH81PoYlFhbGU+6BZp6G7niu735Sk7lN crossorigin=anonymous></script><script src=https://cdn.jsdelivr.net/npm/bootstrap@4.6.1/dist/js/bootstrap.min.js integrity="sha512-UR25UO94eTnCVwjbXozyeVd6ZqpaAE9naiEUBK/A+QDbfSTQFh [...]
\ No newline at end of file