You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@parquet.apache.org by gi...@apache.org on 2025/12/09 19:39:28 UTC
(parquet-site) branch asf-site updated: deploy: b588d89f68fc261905f80474a4175036a005fec7
This is an automated email from the ASF dual-hosted git repository.
github-bot pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/parquet-site.git
The following commit(s) were added to refs/heads/asf-site by this push:
new de0a766 deploy: b588d89f68fc261905f80474a4175036a005fec7
de0a766 is described below
commit de0a766d9993456456cbead6e9222a0f911899c5
Author: emkornfield <em...@users.noreply.github.com>
AuthorDate: Tue Dec 9 19:39:22 2025 +0000
deploy: b588d89f68fc261905f80474a4175036a005fec7
---
output/404.html | 4 +-
output/_print/blog/index.html | 4 +-
output/_print/blog/parquet-format/index.html | 4 +-
output/_print/blog/parquet-java/index.html | 4 +-
output/_print/community/index.html | 4 +-
output/_print/docs/asf/index.html | 4 +-
output/_print/docs/concepts/index.html | 4 +-
.../_print/docs/contribution-guidelines/index.html | 4 +-
.../_print/docs/file-format/data-pages/index.html | 60 +-
output/_print/docs/file-format/index.html | 978 ++++++++++++++++++++-
output/_print/docs/file-format/types/index.html | 825 ++++++++++++++++-
output/_print/docs/index.html | 978 ++++++++++++++++++++-
.../docs/learning-resources/blog-posts/index.html | 4 +-
output/_print/docs/learning-resources/index.html | 4 +-
.../learning-resources/presentations/index.html | 4 +-
output/_print/docs/overview/index.html | 4 +-
output/blog/2019/09/25/2.7.0/index.html | 7 +-
output/blog/2020/01/13/2.8.0/index.html | 7 +-
output/blog/2021/10/06/1.12.2/index.html | 7 +-
output/blog/2021/10/06/2.9.0/index.html | 7 +-
output/blog/2022/05/26/1.12.3/index.html | 7 +-
output/blog/2023/04/06/1.13.0/index.html | 7 +-
output/blog/2023/05/18/1.13.1/index.html | 7 +-
output/blog/2023/11/20/2.10.0/index.html | 6 +-
output/blog/2024/05/07/1.14.0/index.html | 7 +-
output/blog/2024/06/16/1.14.1/index.html | 7 +-
output/blog/2024/08/26/1.14.2/index.html | 7 +-
output/blog/2024/10/05/1.14.3/index.html | 7 +-
output/blog/2024/11/11/1.14.4/index.html | 7 +-
output/blog/2024/12/02/1.15.0/index.html | 7 +-
output/blog/2025/03/16/1.15.1/index.html | 7 +-
output/blog/2025/03/23/2.11.0/index.html | 6 +-
output/blog/2025/05/01/1.15.2/index.html | 7 +-
output/blog/2025/08/28/2.12.0/index.html | 6 +-
output/blog/2025/09/03/1.16.0/index.html | 7 +-
output/blog/index.html | 6 +-
output/blog/page/2/index.html | 6 +-
output/blog/parquet-format/index.html | 6 +-
output/blog/parquet-java/index.html | 6 +-
output/blog/parquet-java/page/2/index.html | 6 +-
output/categories/index.html | 4 +-
output/community/index.html | 4 +-
output/docs/asf/donate/index.html | 6 +-
output/docs/asf/events/index.html | 6 +-
output/docs/asf/index.html | 6 +-
output/docs/asf/license/index.html | 6 +-
output/docs/asf/security/index.html | 6 +-
output/docs/asf/sponsor/index.html | 6 +-
output/docs/concepts/index.html | 7 +-
.../contribution-guidelines/building/index.html | 6 +-
.../contributing/index.html | 7 +-
output/docs/contribution-guidelines/index.html | 6 +-
.../contribution-guidelines/releasing/index.html | 7 +-
.../sub-projects/index.html | 6 +-
.../binaryprotocolextensions}/index.html | 76 +-
output/docs/file-format/bloomfilter/index.html | 16 +-
output/docs/file-format/configurations/index.html | 7 +-
.../file-format/data-pages/checksumming/index.html | 6 +-
.../file-format/data-pages/columnchunks/index.html | 6 +-
.../file-format/data-pages/compression/index.html | 26 +-
.../file-format/data-pages/encodings/index.html | 46 +-
.../file-format/data-pages/encryption/index.html | 46 +-
.../data-pages/errorrecovery/index.html | 6 +-
output/docs/file-format/data-pages/index.html | 8 +-
output/docs/file-format/data-pages/index.xml | 17 +-
output/docs/file-format/extensibility/index.html | 6 +-
.../file-format/implementationstatus/index.html | 7 +-
output/docs/file-format/index.html | 8 +-
output/docs/file-format/index.xml | 22 +-
output/docs/file-format/metadata/index.html | 7 +-
output/docs/file-format/nestedencoding/index.html | 6 +-
output/docs/file-format/nulls/index.html | 6 +-
output/docs/file-format/pageindex/index.html | 26 +-
.../pageindex/src/main/thrift/parquet.thrift | 16 +
.../geospatial}/index.html | 76 +-
output/docs/file-format/types/index.html | 8 +-
output/docs/file-format/types/index.xml | 36 +-
.../docs/file-format/types/logicaltypes/index.html | 397 ++++++++-
.../file-format/types/variantencoding/index.html | 247 ++++++
.../file-format/types/variantshredding}/index.html | 186 +++-
output/docs/index.html | 6 +-
.../docs/learning-resources/blog-posts/index.html | 6 +-
output/docs/learning-resources/index.html | 6 +-
.../conf-2014-parquet-summit-twitter/index.html | 6 +-
.../presentations/hadoop-summit-2014/index.html | 6 +-
.../learning-resources/presentations/index.html | 6 +-
.../presentations/spark-summit-2020/index.html | 6 +-
.../presentations/strata-2013/index.html | 6 +-
output/docs/overview/index.html | 7 +-
output/docs/overview/motivation/index.html | 6 +-
output/index.html | 4 +-
output/index.xml | 75 +-
...474ec609632241944fd383a1b8a5f849b29478b9283a.js | 5 +
.../doc/images/FileLayoutBloomFilter1.png | Bin 0 -> 44025 bytes
.../doc/images/FileLayoutBloomFilter2.png | Bin 0 -> 34018 bytes
.../doc/images/FileLayoutEncryptionEF.png | Bin 0 -> 81119 bytes
.../doc/images/FileLayoutEncryptionPF.png | Bin 0 -> 79304 bytes
.../parquet-format/doc/images/PageIndexLayout.png | Bin 0 -> 7442 bytes
...aeb4abff48eff01271de19b46b867859ce3df1146de.css | 9 +
output/sitemap.xml | 2 +-
output/tags/index.html | 4 +-
101 files changed, 4134 insertions(+), 472 deletions(-)
diff --git a/output/404.html b/output/404.html
index 8702168..f448f1c 100644
--- a/output/404.html
+++ b/output/404.html
@@ -1,3 +1,3 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/blog/index.html b/output/_print/blog/index.html
index cb0ffa6..105f277 100644
--- a/output/_print/blog/index.html
+++ b/output/_print/blog/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180>< [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180>< [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/blog/>Return to the regular view of this page</a>.</p></div><h1 class=title>Releases</h1><ul><li><a href=#pg-a9373d016caafc4ca68331dc6fa974ca>Parquet Format Releases</a></li><ul><li><a href=#pg-c5d05c7d6a0a2e1eeb558e4983210334>2.12.0</a></li><li><a href=#pg-099976801d2e02655971d557c9d53a43>2.11.0</a></li><li><a href=#pg-c9e50ac968526979f7dbe2558d97e2b0>2.10.0</a></li><li><a href=#pg-e942f5058827113b1d4e123b925d [...]
...
<dependency>
@@ -118,4 +118,4 @@
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=toolt [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/blog/parquet-format/index.html b/output/_print/blog/parquet-format/index.html
index c4d8616..32670a8 100644
--- a/output/_print/blog/parquet-format/index.html
+++ b/output/_print/blog/parquet-format/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/blog/parquet-format/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ic [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/blog/parquet-format/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ic [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/blog/parquet-format/>Return to the regular view of this page</a>.</p></div><h1 class=title>Parquet Format Releases</h1><ul><li><a href=#pg-c5d05c7d6a0a2e1eeb558e4983210334>2.12.0</a></li><li><a href=#pg-099976801d2e02655971d557c9d53a43>2.11.0</a></li><li><a href=#pg-c9e50ac968526979f7dbe2558d97e2b0>2.10.0</a></li><li><a href=#pg-e942f5058827113b1d4e123b925d385b>2.9.0</a></li><li><a href=#pg-51204f9eba87d824bc1f [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/blog/parquet-java/index.html b/output/_print/blog/parquet-java/index.html
index f715d5a..fbd039c 100644
--- a/output/_print/blog/parquet-java/index.html
+++ b/output/_print/blog/parquet-java/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/blog/parquet-java/><link rel=alternate type=application/rss+xml href=/blog/parquet-java/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-1 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/blog/parquet-java/><link rel=alternate type=application/rss+xml href=/blog/parquet-java/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-1 [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/blog/parquet-java/>Return to the regular view of this page</a>.</p></div><h1 class=title>Parquet-Java Releases</h1><ul><li><a href=#pg-a6de0fb1dd3d04eff087f9d7f936dbd7>1.16.0</a></li><li><a href=#pg-1e7b153e7a8d50d1635c75f665be23db>1.15.2</a></li><li><a href=#pg-aa23966cf978a1a5daade90632e1545b>1.15.1</a></li><li><a href=#pg-f97b49e01c16973f915495097260f835>1.15.0</a></li><li><a href=#pg-57c1244ba5ea438c674c986 [...]
...
<dependency>
@@ -118,4 +118,4 @@
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=toolt [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/community/index.html b/output/_print/community/index.html
index 4f88456..4d6c11d 100644
--- a/output/_print/community/index.html
+++ b/output/_print/community/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes [...]
<a href=https://www.apache.org>Apache Software Foundation</a>
and is developed and maintained by a contributors from around the world.
We'd love you to join us!</p></span></div></div></section><section class="row td-box td-box--white td-box--height-auto linkbox"><div class="col-xs-12 col-sm-6 col-md-6 col-lg-6"><h2>Learn and Connect</h2><p><b>Development Mailing List</b>
@@ -20,4 +20,4 @@ for agendas of past meetings.</p><ul><li title=Subscribe><a href=https://groups.
Join the Google group to access the calendar invite, agenda, and recordings of past calls.</li></ul></div><div class="col-xs-12 col-sm-6 col-md-6 col-lg-6"><h2>Develop and Contribute</h2><p>If you want to get more involved by contributing to Parquet, please see our
<a href=/docs/contribution-guidelines/>Developer Guide.</a></p></div></section></main><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener href=mailto:dev@parquet.apache.org aria-label="Post to Mailing List"><i clas [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/asf/index.html b/output/_print/docs/asf/index.html
index 99f0617..54d4355 100644
--- a/output/_print/docs/asf/index.html
+++ b/output/_print/docs/asf/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=1 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=1 [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/asf/>Return to the regular view of this page</a>.</p></div><h1 class=title>Apache Software Foundation (ASF)</h1><div class=lead>Apache Software Foundation</div><ul><li>1: <a href=#pg-57942ee0c37ed732b8a36ad6da469b9b>License</a></li><li>2: <a href=#pg-4a41ca42a224dc588a84f6aceb800983>Security</a></li><li>3: <a href=#pg-18eeebb45e0fdfd01843649980749ac6>Sponsor</a></li><li>4: <a href=#pg-f237f8bebf4f33fc4aedb [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/concepts/index.html b/output/_print/docs/concepts/index.html
index 45f9d07..cae9338 100644
--- a/output/_print/docs/concepts/index.html
+++ b/output/_print/docs/concepts/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/concepts/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.p [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/concepts/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.p [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/concepts/>Return to the regular view of this page</a>.</p></div><h1 class=title>Concepts</h1><div class=lead>Glossary of relevant terminology.</div><ul></ul><div class=content><ul><li><p><em>Block (HDFS block)</em>: This means a block in HDFS and the meaning is
unchanged for describing this file format. The file format is
designed to work well on top of HDFS.</p></li><li><p><em>File</em>: A HDFS file that must include the metadata for the file.
@@ -11,4 +11,4 @@ be multiple page types which are interleaved in a column chunk.</p></li></ul><p>
contains exactly one column chunk per column. Column chunks contain one or
more pages.</p><h2 id=unit-of-parallelization>Unit of parallelization</h2><ul><li>MapReduce - File/Row Group</li><li>IO - Column chunk</li><li>Encoding/Compression - Page</li></ul></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Pos [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/contribution-guidelines/index.html b/output/_print/docs/contribution-guidelines/index.html
index b6eb43e..2f124ce 100644
--- a/output/_print/docs/contribution-guidelines/index.html
+++ b/output/_print/docs/contribution-guidelines/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/contribution-guidelines/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favic [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/contribution-guidelines/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favic [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/contribution-guidelines/>Return to the regular view of this page</a>.</p></div><h1 class=title>Developer Guide</h1><div class=lead>All developer resources related to Parquet.</div><ul><li>1: <a href=#pg-84c5df6519663fa8413f5c392f10dbd4>Sub-Projects</a></li><li>2: <a href=#pg-0fc7677c5a8dcd5250334bbf678cb165>Building Parquet</a></li><li>3: <a href=#pg-47cac26307c77b16f1b9e75c1e46efec>Contributing to Parquet [...]
Java resources can be build using <code>mvn package</code>. The current stable version should always be available from Maven Central.</p><p>C++ thrift resources can be generated via make.</p><p>Thrift can be also code-genned into any other thrift-supported language.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-47cac26307c77b16f1b9e75c1e46efec>3 - Contributing to Parquet-Java</h1><div class=lead>How to contribute to Parquet-Java</div><h2 id=pull-requests>Pull Re [...]
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * @param c the current class
@@ -81,4 +81,4 @@ releases on a quarterly basis (Targets months are January, April, July and
October). If a new major version is necessary it will be targetted for the
October release.</p></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener href=mailto:dev@parquet.apache.org aria-label="Post to Mailing List"><i class="fa fa-envelope"></i></a></li><li class [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/file-format/data-pages/index.html b/output/_print/docs/file-format/data-pages/index.html
index 03d354d..79fca1f 100644
--- a/output/_print/docs/file-format/data-pages/index.html
+++ b/output/_print/docs/file-format/data-pages/index.html
@@ -6,18 +6,18 @@ The encoded values for the data page is always required. The definition and repe
repetition levels data definition levels data encoded values The value of uncompressed_page_size specified in the header is for all the 3 pieces combined.
The encoded values for the data page is always required. The definition and repetition levels are optional, based on the schema definition. If the column is not nested (i.e. the path to the column has length 1), we do not encode the repetition levels (it would always have the value 1). For data that is required, the definition levels are skipped (if encoded, it will always have the value of the max definition level)."><meta itemprop=dateModified content="2024-01-14T20:32:15+08:00"><meta [...]
repetition levels data definition levels data encoded values The value of uncompressed_page_size specified in the header is for all the 3 pieces combined.
-The encoded values for the data page is always required. The definition and repetition levels are optional, based on the schema definition. If the column is not nested (i.e. the path to the column has length 1), we do not encode the repetition levels (it would always have the value 1). For data that is required, the definition levels are skipped (if encoded, it will always have the value of the max definition level)."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca [...]
-<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/data-pages/>Return to the regular view of this page</a>.</p></div><h1 class=title>Data Pages</h1><ul><li>1: <a href=#pg-1745a7620ee7ae9962c3e6baaa083e89>Compression</a></li><li>2: <a href=#pg-9aa971e751fdd370158d525ad337ef7a>Encodings</a></li><li>3: <a href=#pg-70f861f07d9aae8f8d133f3351caad43>Parquet Modular Encryption</a></li><li>4: <a href=#pg-e4790b30bda91ce23679bbaa9c652cd0>Checksumming</a [...]
+The encoded values for the data page is always required. The definition and repetition levels are optional, based on the schema definition. If the column is not nested (i.e. the path to the column has length 1), we do not encode the repetition levels (it would always have the value 1). For data that is required, the definition levels are skipped (if encoded, it will always have the value of the max definition level)."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48e [...]
+<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/data-pages/>Return to the regular view of this page</a>.</p></div><h1 class=title>Data Pages</h1><ul><li>1: <a href=#pg-1745a7620ee7ae9962c3e6baaa083e89></a></li><li>2: <a href=#pg-9aa971e751fdd370158d525ad337ef7a></a></li><li>3: <a href=#pg-70f861f07d9aae8f8d133f3351caad43></a></li><li>4: <a href=#pg-e4790b30bda91ce23679bbaa9c652cd0>Checksumming</a></li><li>5: <a href=#pg-d84ba941c88c28699d637 [...]
header. No padding is allowed in the data page.
In order we have:</p><ol><li>repetition levels data</li><li>definition levels data</li><li>encoded values</li></ol><p>The value of <code>uncompressed_page_size</code> specified in the header is for all the 3 pieces combined.</p><p>The encoded values for the data page is always required. The definition and repetition levels
are optional, based on the schema definition. If the column is not nested (i.e.
the path to the column has length 1), we do not encode the repetition levels (it would
always have the value 1). For data that is required, the definition levels are
skipped (if encoded, it will always have the value of the max definition level).</p><p>For example, in the case where the column is non-nested and required, the data in the
-page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p></div></div><div class=td-content style=page-break-before:always><h1 id=pg-1745a7620ee7ae9962c3e6baaa083e89>1 - Compression</h1><h2 id=overview>Overview</h2><p>Parquet allows the data block inside dictionary pages and data pages to
+page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p></div></div><div class=td-content style=page-break-before:always><h1 id=pg-1745a7620ee7ae9962c3e6baaa083e89>1 -</h1><h1 id=parquet-compression-definitions>Parquet compression definitions</h1><p>This document contains the specification of all supported compression codecs.</p><h2 id=overview>Overview</h2><p>Parquet allows the [...]
be compressed for better space efficiency. The Parquet format supports
-several compression covering different areas in the compression ratio /
-processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
+several compression codecs covering different areas in the compression
+ratio / processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
by their respective authors or maintainers, which we reference hereafter.</p><p>For all compression codecs except the deprecated <code>LZ4</code> codec, the raw data
of a (data or dictionary) page is fed <em>as-is</em> to the underlying compression
library, without any additional framing or padding. The information required
@@ -47,8 +47,8 @@ when implementing this format, the implementation provided by the
<a href=https://facebook.github.io/zstd/>Zstandard compression library</a>
is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md>LZ4 block format</a>.
If any ambiguity arises when implementing this format, the implementation
-provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>2 - Encodings</h1><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supported for types. It is
-intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding cannot be used. It
+provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>2 -</h1><h1 id=parquet-encoding-definitions>Parquet encoding definitions</h1><p>This file contains the specification of all supported encodings.</p><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supporte [...]
+intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
@@ -75,8 +75,7 @@ repeated-value := value that is repeated, using a fixed-width of round-up-to-nex
</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
-significant. An example of the encoding is presented below. For comparison, the same case is shown
-in the example of the deprecated bit-packing encoding in the next section.</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 10001000 11000110 11111010
@@ -150,7 +149,9 @@ by keeping track of the number of values read.</p><p>Subtractions in steps 1) an
will the corresponding additions when decoding. Overflow should be allowed
and handled as wrapping around in 2’s complement notation so that the original
values are correctly restituted. This may require explicit care in some programming
-languages (for example by doing all arithmetic in the unsigned domain).</p><p>The following examples use 8 as the block size to keep the examples short,
+languages (for example by doing all arithmetic in the unsigned domain). Writers
+must not use more bits when bit packing the miniblock data than would be required
+to PLAIN encode the physical type (e.g. INT32 data must not use more than 32 bits).</p><p>The following examples use 8 as the block size to keep the examples short,
but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2, 3, 4, 5</p><p>After step 1), we compute the deltas as:</p><p>1, 1, 1, 1</p><p>The minimum delta is 1 and after step 2, the relative deltas become:</p><p>0, 0, 0, 0</p><p>The final encoded data is:</p><p>header:
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
@@ -162,11 +163,10 @@ concatenated back to back. The expected savings is from the cost of encoding the
and possibly better compression in the data (it is no longer interleaved with the lengths).</p><p>The data stream looks like:</p><pre tabindex=0><code><Delta Encoded Lengths> <Byte Array Data>
</code></pre><p>For example, if the data was “Hello”, “World”, “Foobar”, “ABCDEF”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(5, 5, 6, 6) (the string lengths)</li><li>“HelloWorldFoobarABCDEF”</li></ul><h3 id=delta-strings-delta_byte_array--7>Delta Strings: (DELTA_BYTE_ARRAY = 7)</h3><p>Supported Types: BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY</p><p>This is also known as incremental [...]
sequence of strings, store the prefix length of the previous entry plus the suffix.</p><p>For a longer description, see <a href=https://en.wikipedia.org/wiki/Incremental_encoding>https://en.wikipedia.org/wiki/Incremental_encoding</a>.</p><p>This is stored as a sequence of delta-encoded prefix lengths (DELTA_BINARY_PACKED), followed by
-the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
+the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
compression ratio and speed when a compression algorithm is used afterwards.</p><p>This encoding creates K byte-streams of length N where K is the size in bytes of the data
-type and N is the number of elements in the data sequence. Specifically, K is 4 for FLOAT
-type and 8 for DOUBLE type.
-The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
+type and N is the number of elements in the data sequence. For example, K is 4 for FLOAT
+type and 8 for DOUBLE type.</p><p>The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
0-th stream, the 1-st byte goes to the 1-st stream and so on.
The streams are concatenated in the following order: 0-th stream, 1-st stream, etc.
The total length of encoded streams is K * N bytes. Because it does not have any metadata
@@ -175,7 +175,7 @@ is allowed inside the data page.</p><p>Example:
Original data is three 32-bit floats and for simplicity we look at their raw representation.</p><pre tabindex=0><code> Element 0 Element 1 Element 2
Bytes AA BB CC DD 00 11 22 33 A3 B4 C5 D6
</code></pre><p>After applying the transformation, the data has the following representation:</p><pre tabindex=0><code>Bytes AA 00 A3 BB 11 B4 CC 22 C5 DD 33 D6
-</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-70f861f07d9aae8f8d133f3351caad43>3 - Parquet Modular Encryption</h1><p>Parquet files containing sensitive information can be protected by the modular encryption
+</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-70f861f07d9aae8f8d133f3351caad43>3 -</h1><h1 id=parquet-modular-encryption>Parquet Modular Encryption</h1><p>Parquet files containing sensitive information can be protected by the modular encryption
mechanism that encrypts and authenticates the file data and metadata - while allowing
for a regular Parquet functionality (columnar projection, predicate pushdown, encoding
and compression).</p><h2 id=1-problem-statement>1 Problem Statement</h2><p>Existing data protection solutions (such as flat encryption of files, in-storage encryption,
@@ -230,7 +230,20 @@ The unique input to CTR encryption is called IV (“initialization vector&rd
parts: a nonce and an initial counter field.</p><p>Parquet encryption uses the RBG-based (random bit generator) nonce construction as defined in
the section 8.2.2 of the NIST SP 800-38D document. For each encrypted module, Parquet generates a
unique nonce with a length of 12 bytes (96 bits). Notice: the NIST
-specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
+specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h4 id=414-invocation-limit>4.1.4 Invocation limit</h4><p>According to the section 8.3 of the NIST SP 800-38D document, <em>“The total number of invocations
+of the authenticated encryption function shall not exceed 2^32, including all IV lengths and
+all instances of the authenticated encryption function with the given key”</em>. This restriction is
+related to the “uniqueness requirement of IVs and keys” (section 8 in the NIST spec) - <em>“if even
+one IV is ever repeated, then the implementation may be vulnerable”</em>. <em>“Compliance with this
+requirement is crucial to the security of GCM”</em>.</p><p>The bulk of modules in a Parquet file are page headers and data pages. Therefore, one encryption
+key shall not not be used for more than 2^31 (~2 billion) pages. In Parquet files encrypted with
+multiple keys (footer and column keys), the constraint on the number of invocations is applied
+to each key separately.</p><p>When running in the context of a larger system, any particular Parquet writer implementation likely
+does not have sufficient context to enforce key invocation limits system-wide. Therefore,
+the higher level system itself must arrange to supply keys appropriately to the various writer instances.</p><p>Parquet writer implementations should have a local invocation counter for each encryption key. If the
+counter exceeds 2^32, the implementation should return an error and produce no more cipherblocks.
+While this does not enforce a system-wide limit, it helps in distributed systems that provide different
+keys to different nodes (or generate unique keys in each node).</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
must be implemented by a cryptographic provider according to the NIST SP 800-38D specification.</p><p>In Parquet, an input to the GCM cipher is an encryption key, a 12-byte nonce, a plaintext and an
AAD. The output is a ciphertext with the length equal to that of plaintext, and a 16-byte authentication
tag used to verify the ciphertext and AAD integrity.</p><h4 id=422-aes_gcm_ctr_v1>4.2.2 AES_GCM_CTR_V1</h4><p>In this Parquet algorithm, all modules except pages are encrypted with the GCM cipher, as described
@@ -277,7 +290,7 @@ data set (table). This string is optionally passed by a writer upon file creatio
the AAD prefix is stored in an <code>aad_prefix</code> field in the file, and is made available to the readers.
This field is not encrypted. If a user is concerned about keeping the file identity inside the file,
the writer code can explicitly request Parquet not to store the AAD prefix. Then the aad_prefix field
-will be empty; AAD prefixes must be fully managed by the caller code and supplied explicitly to Parquet
+will be empty; AAD prefixes must be fully managed by the caller code and supplied explictly to Parquet
readers for each file.</p><p>The protection against swapping full files is optional. It is not enabled by default because
it requires the writers to generate and pass an AAD prefix.</p><p>A reader of a file created with an AAD prefix, should be able to verify the prefix (file identity)
by comparing it with e.g. the target table name, using a convention accepted in the organization.
@@ -333,7 +346,10 @@ decrypt it.</p><p>The row group ordinal, required for AAD suffix calculation, is
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/** Row group ordinal in the file **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>i16</span> <span style=color:#000>ordinal</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
+</span></span></code></pre></div><p>The integrity of this field is protected by authenticated encryption of the footer (FileMetaData). Therefore,
+the reader implementations can use either a local row group counter (ordinal) or the <code>RowGroup.ordinal</code>
+field as an input to AAD suffix calculation. The latter option can be helpful when different reader
+threads process different row groups in the same parquet file.</p><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
is a union - the actual structure is chosen depending on whether the column is encrypted with the
footer encryption key, or with a column-specific key. For the latter, a key metadata can be specified.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span> <span style=color:#000>EncryptionWithFooterKey</span> <span style=color:#000;font-weight:700>{</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
@@ -407,7 +423,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img src=/parquet-format/doc/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -430,7 +446,7 @@ structure with the
AES GCM algorithm - using a footer signing key, and an AAD constructed according to the instructions
of the section 4.4. Only the nonce and GCM tag are stored in the file – as a 28-byte
fixed-length array, written right after the footer itself. The ciphertext is not stored,
-because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</ [...]
+because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>stru [...]
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>...</span>
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/**
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * Encryption algorithm. This field is set only in encrypted files
@@ -451,7 +467,7 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img src=/parquet-format/doc/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
@@ -482,4 +498,4 @@ Each file metadata would be cumulative and include all the row groups written so
far. Combining this with the strategy used for rc or avro files using sync markers,
a reader could recover partially written files.</p></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener href=mailto:dev@parquet.apache.org aria-label="Post to Mailing List"><i class="fa fa-e [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/file-format/index.html b/output/_print/docs/file-format/index.html
index cdd9014..8fc776f 100644
--- a/output/_print/docs/file-format/index.html
+++ b/output/_print/docs/file-format/index.html
@@ -1,5 +1,5 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/file-format/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180 [...]
-<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/>Return to the regular view of this page</a>.</p></div><h1 class=title>File Format</h1><div class=lead>Documentation about the Parquet File Format.</div><ul><li>1: <a href=#pg-de128b03a7277c17c6dec2cef97af32d>Configurations</a></li><li>2: <a href=#pg-6762d78210357c1df172dfddcc6fd307>Extensibility</a></li><li>3: <a href=#pg-e1f24cbf7bc8c4ddd1cc8213993795a7>Metadata</a></li><li>4: <a href=#pg-140 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/file-format/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180 [...]
+<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/>Return to the regular view of this page</a>.</p></div><h1 class=title>File Format</h1><div class=lead>Documentation about the Parquet File Format.</div><ul><li>1: <a href=#pg-a8f87c969a3621c87488ade11fb64f28></a></li><li>2: <a href=#pg-de128b03a7277c17c6dec2cef97af32d>Configurations</a></li><li>3: <a href=#pg-6762d78210357c1df172dfddcc6fd307>Extensibility</a></li><li>4: <a href=#pg-e1f24cbf7bc [...]
<Column 1 Chunk 1>
<Column 2 Chunk 1>
...
@@ -22,7 +22,70 @@ start locations. More details on what is contained in the metadata can be found
in the Thrift definition.</p><p>File metadata is written after the data to allow for single pass writing.</p><p>Readers are expected to first read the file metadata to find all the column
chunks they are interested in. The columns chunks should then be read sequentially.</p><p>The format is explicitly designed to separate the metadata from the data. This
allows splitting columns into multiple files, as well as having a single metadata
-file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p></div></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>1 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
+file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p></div></div><div class=td-content style=page-break-before:always><h1 id=pg-a8f87c969a3621c87488ade11fb64f28>1 -</h1><h1 id=binary-protocol-extensions>Binary Protocol Extensions</h1><p>The extension mechanism of the <code>binary</code> Thrift field-id <code>32767</code> has some desirable properties:</p><ul><li>Existing readers will ignore these extensions without any modifications</li><li>E [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(M) encoded size of the extension
+M bytes | extension bytes
+1 byte | \0 (thrift stop field)
+</code></pre><p>The choice to reserve only one field-id has an additional (and frankly unintended) property. It creates scarcity in the extension space and disincentivizes vendors from keeping their extensions private. As a vendor having an extension means one cannot use it in tandem with other extensions from other vendors even if such extensions are publicly known. The easiest path of interoperability and ability to further experiment is to push an extension through standardization and [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(K+28) encoded size of the extension
+K bytes | Flatbuffers representation (v0) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+16 bytes | some-UUID
+1 byte | \0 (thrift stop field)
+4 bytes | PAR1
+</code></pre><p>some-UUID is some UUID picked for this extension and it is used throughout (possibly internal) experimentation. It is put at the end to allow detection of the extension when parsed in reverse. The little-endian sizes and crc32s are also to the end to facilitate efficient parsing the footer in reverse without requiring parsing the Thrift compact protocol that precedes it.</p><p>At some point the experiments conclude and the extension shared publicly with the community. The [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(K+28) encoded size of the extension
+K bytes | Flatbuffers representation (v1) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+16 bytes | some-other-UUID
+1 byte | \0 (thrift stop field)
+4 bytes | PAR1
+</code></pre><p>After the migration period, the end of the Parquet file may look like this:</p><pre><code>K bytes | Flatbuffers representation (v1) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+4 bytes | PAR3
+</code></pre><p>In this example, we see several design decisions for the extension at play:</p><ul><li>There is a new some-other-UUID for the accepted change to the standard and now the Thrift <code>FileMetaData</code> cannot be extended itself.</li><li>The length of the footer and the crc32 of the length itself, guarantees that new readers will not overshoot reading bytes in case of corrupt bits in these critical 8 bytes of the file.</li><li>The crc32 of the flatbuffer representation en [...]
+ | | Column b (new encoding)
+ | | Column c (new encoding)
+R bytes | Row Group | Column a
+ | 0 | Column d
+ | | Column b (old encoding)
+ | | Column c (old encoding)
+ | | FileMetaData
+ | | ColumnMetaData: a
+ | | ColumnMetaData: b
+F bytes | | <extension-blob with offsets to new encoding>
+ | | ColumnMetaData: c
+ | | <extension-blob with offsets to new encoding>
+ | | ColumnMetaData: d
+4 bytes | PAR1
+</code></pre><p>The custom reader is compiled with thrift IDL with a binary for field with id 32767. This is done to become extension aware and inspect the extension bytes looking for the UUID disambiguator. If that’s found it decodes the offsets from the rest of the bytes and reads the region of the file containing the new encoding.</p><p>If/when the encoding is ratified, it is added to the official specification as an additional type in <code>Encodings</code> at which point the extensi [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>while</span> <span style=color:#000;font-weight:700>(</span><span style=color:#204a87>true</span><span style=color:#000;font-weight:700>)</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>uint8_t</span> <span style=color:#000>c</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700>&</span> <span style=color:#0000cf;font-weight:700>0x7F</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700><</span> <span style=color:#0000cf;font-weight:700>0x80</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>out</span><span style=color:#ce5c00;font-weight:700>-></span><span style=co [...]
+</span></span><span style=display:flex><span> <span style=color:#000>out</span><span style=color:#ce5c00;font-weight:700>-></span><span style=color:#000>push_back</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>c</span> <span style=color:#ce5c00;font-weight:700>+</span> <span style=color:#0000cf;font-weight:700>0x80</span><span style=color:#000;font-weight:700>);</span>
+</span></span><span style=display:flex><span> <span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700>>>=</span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>};</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#000>std</span><span style=color:#ce5c00;font-weight:700>::</span><span style=color:#000>string</span> <span style=color:#000>AppendExtension</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>std</span><span style=color:#ce5c00;font-weight:700>::</span><span style=color:#000>string</span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;fon [...]
+</span></span><span style=display:flex><span> <span style=color:#000>assert</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>back</span><span style=color:#000;font-weight:700>()</span> <span style=color:#ce5c00;font-weight:700>==</span> <span style=color:#4e9a06>'\x00'</span><span style=color:#000;font-weight:700>);</span> <span style=color:#8f5902;font-style:italic>// [...]
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>back</span><span style=color:#000;font-weight:700>()</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#4e9a06>'\x08'</span><span style=color:#000;font-weight:700>;</span> <span style=color:#8f5902;font-style:italic>// replace stop field with [...]
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AppendUleb</span><span style=color:#000;font-weight:700>(</span><span style=color:#0000cf;font-weight:700>32767</span><span style=color:#000;font-weight:700>,</span> <span style=color:#ce5c00;font-weight:700>&</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>);</span> <span style=color:#8f5902;font-style:italic>// field-id
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AppendUleb</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>ext</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>size</span><span style=color:#000;font-weight:700>(),</span> <span style=color:#ce5c00;font-weight:700>&</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>);</span>
+</span></span><span style=display:flex><span> <span style=color:#000>thrift</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#000>ext</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#000>thrift</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#4e9a06>'\x00'</span><span style=color:#000;font-weight:700>;</span> <span style=color:#8f5902;font-style:italic>// add the stop field
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
+</span></span></code></pre></div></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>2 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
possible to do larger sequential IO. Larger groups also require more buffering in
the write path (or a two pass write). We recommend large row groups (512MB - 1GB).
Since an entire row group might need to be read, we want it to completely fit on
@@ -32,12 +95,12 @@ per HDFS file.</p><h3 id=data-page--size>Data Page Size</h3><p>Data pages should
allow for more fine grained reading (e.g. single row lookup). Larger page sizes
incur less space overhead (less page headers) and potentially less parsing overhead
(processing headers). Note: for sequential scans, it is not expected to read a page
-at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-6762d78210357c1df172dfddcc6fd307>2 - Extensibility</h1><p>There are many places in the format for compatible extensions:</p><ul><li>File Version: The file metadata contains a version.</li><li>Encodings: Encodings are specified by enum and more can be added in the future.</li><li>Page types: Additional page types can be added and safely skipped.</li [...]
+at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-6762d78210357c1df172dfddcc6fd307>3 - Extensibility</h1><p>There are many places in the format for compatible extensions:</p><ul><li>File Version: The file metadata contains a version.</li><li>Encodings: Encodings are specified by enum and more can be added in the future.</li><li>Page types: Additional page types can be added and safely skipped.</li [...]
definition of these structures is given in the Parquet
<a href=https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift>Thrift definition</a>.</p><h2 id=file-metadata>File metadata</h2><p>In the diagram below, file metadata is described by the <code>FileMetaData</code>
structure. This file metadata provides offset and size information useful
when navigating the Parquet file.</p><p><img src=/images/FileMetaData.svg alt="Parquet Metadata format"></p><h2 id=page-header>Page header</h2><p>Page header metadata (<code>PageHeader</code> and children in the diagram) is stored
-in-line with the page data, and is used in the reading and decoding of data.</p><p><img src=/images/PageHeader.svg alt="Parquet PageHeader format"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>4 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
+in-line with the page data, and is used in the reading and decoding of data.</p><p><img src=/images/PageHeader.svg alt="Parquet PageHeader format"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>5 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
32-bit ints with an efficient encoding. This reduces the complexity of implementing
@@ -49,19 +112,828 @@ readers and writers for the format. The types are:</p><pre tabindex=0><code> -
- DOUBLE: IEEE 64-bit floating point values
- BYTE_ARRAY: arbitrarily long byte arrays
- FIXED_LEN_BYTE_ARRAY: fixed length byte arrays
-</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-967cf8be84f5efbfd2e57475707b79b1>4.1 - Logical Types</h1><p>Logical types are used to extend the types that parquet can be used to store,
+</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-95b2ce86b5e36722017e238af2bd0fd8>5.1 -</h1><h1 id=geospatial-definitions>Geospatial Definitions</h1><p>This document contains the specification of geospatial types and statistics.</p><h1 id=background>Background</h1><p>The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and
+Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM,
+XYZM) are defined by <a href="https://portal.ogc.org/files/?artifact_id=25355">OpenGIS Implementation Specification for Geographic
+information - Simple feature access - Part 1: Common architecture</a>,
+from <a href=https://www.ogc.org/standard/sfa/>OGC(Open Geospatial Consortium)</a>.</p><p>The version of the OGC standard first used here is 1.2.1, but future versions
+may also be used if the WKB representation remains wire-compatible.</p><h2 id=coordinate-reference-system>Coordinate Reference System</h2><p>Coordinate Reference System (CRS) is a mapping of how coordinates refer to
+locations on Earth.</p><p>The default CRS <code>OGC:CRS84</code> means that the geospatial features must be stored
+in the order of longitude/latitude based on the WGS84 datum.</p><p>Custom CRS can be specified by a string value. It is recommended to use an
+identifier-based approach like <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>.</p><p>For geographic CRS, longitudes are bound by [-180, 180] and latitudes are bound
+by [-90, 90].</p><h2 id=edge-interpolation-algorithm>Edge Interpolation Algorithm</h2><p>An algorithm for interpolating edges, and is one of the following values:</p><ul><li><code>spherical</code>: edges are interpolated as geodesics on a sphere.</li><li><code>vincenty</code>: <a href=https://en.wikipedia.org/wiki/Vincenty%27s_formulae>https://en.wikipedia.org/wiki/Vincenty%27s_formulae</a></li><li><code>thomas</code>: Thomas, Paul D. Spheroidal geodesics, reference systems, & local geom [...]
+logical types to store statistics of a column chunk. It is an optional field in
+the <code>ColumnMetaData</code> and contains <a href=/docs/file-format/types/geospatial/#bounding-box>Bounding Box</a> and <a href=/docs/file-format/types/geospatial/#geospatial-types>Geospatial
+Types</a> that are described below in detail.</p><h2 id=bounding-box>Bounding Box</h2><p>A geospatial instance has at least two coordinate dimensions: X and Y for 2D
+coordinates of each point. Please note that X is longitude/easting and Y is
+latitude/northing. A geospatial instance can optionally have Z and/or M values
+associated with each point.</p><p>The Z values introduce the third dimension coordinate. Usually they are used to
+indicate the height, or elevation.</p><p>M values are an opportunity for a geospatial instance to track a value in a
+fourth dimension. These values can be used as a linear reference value (e.g.,
+highway milepost value), a timestamp, or some other value as defined by the CRS.</p><p>Bounding box is defined as the thrift struct below in the representation of
+min/max value pair of coordinates from each axis. Note that X and Y Values are
+always present. Z and M are omitted for 2D geospatial instances.</p><p>When calculating a bounding box, null or NaN values in a coordinate
+dimension are skipped. For example, <code>POINT (1 NaN)</code> contributes a value to X
+but no values to Y, Z, or M dimension of the bounding box. If a dimension has
+only null or NaN values, that dimension is omitted from the bounding box. If
+either the X or Y dimension is missing, then the bounding box itself is not
+produced.</p><p>For the X values only, xmin may be greater than xmax. In this case, an object
+in this bounding box may match if it contains an X such that <code>x >= xmin</code> OR
+<code>x <= xmax</code>. This wraparound occurs only when the corresponding bounding box
+crosses the antimeridian line. In geographic terminology, the concepts of <code>xmin</code>,
+<code>xmax</code>, <code>ymin</code>, and <code>ymax</code> are also known as <code>westernmost</code>, <code>easternmost</code>,
+<code>southernmost</code> and <code>northernmost</code>, respectively.</p><p>For <code>GEOGRAPHY</code> types, X and Y values are restricted to the canonical ranges of
+[-180, 180] for X and [-90, 90] for Y.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-thrift data-lang=thrift><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>BoundingBox</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000;font-weight:700>{</span><s [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>2</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>3</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>4</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>5</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>6</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>8</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#000;font-weight:700>}</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span></code></pre></div><h2 id=geospatial-types>Geospatial Types</h2><p>A list of geospatial types from all instances in the <code>GEOMETRY</code> or <code>GEOGRAPHY</code>
+column, or an empty list if they are not known.</p><p>This is borrowed from <a href="https://github.com/opengeospatial/geoparquet/blob/v1.1.0/format-specs/geoparquet.md?plain=1#L159">geometry_types of GeoParquet</a> except that
+values in the list are <a href=https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary>WKB (ISO-variant) integer codes</a>.
+Table below shows the most common geospatial types and their codes:</p><table><thead><tr><th style=text-align:left>Type</th><th style=text-align:left>XY</th><th style=text-align:left>XYZ</th><th style=text-align:left>XYM</th><th style=text-align:center>XYZM</th></tr></thead><tbody><tr><td style=text-align:left>Point</td><td style=text-align:left>0001</td><td style=text-align:left>1001</td><td style=text-align:left>2001</td><td style=text-align:center>3001</td></tr><tr><td style=text-alig [...]
+responsible for serializing and deserializing the CRS, respectively.</p><p>As a convention to maximize the interoperability, custom CRS values can be
+specified by a string of the format <code>type:identifier</code>, where <code>type</code> is one of
+the following values:</p><ul><li><code>srid</code>: <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>, <code>identifier</code> is the SRID itself.</li><li><code>projjson</code>: <a href=https://proj.org/en/stable/specifications/projjson.html>PROJJSON</a>, <code>identifier</code> is the name of a table property or a file property where the projjson string is stored.</li></ul><h1 id=coordinate-axis-order>Coordinate axis order</h1><p> [...]
+follows the de facto standard for axis order in WKB and is therefore always
+(x, y) where x is easting or longitude and y is northing or latitude. This
+ordering explicitly overrides the axis order as specified in the CRS.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-967cf8be84f5efbfd2e57475707b79b1>5.2 -</h1><h1 id=parquet-logical-type-definitions>Parquet Logical Type Definitions</h1><p>Logical types are used to extend the types that parquet can be used to store,
by specifying how the primitive types should be interpreted. This keeps the set
of primitive types to a minimum and reuses parquet’s efficient encodings. For
-example, strings are stored as byte arrays (binary) with a UTF8 annotation.
-These annotations define how to further decode and interpret the data.
-Annotations are stored as <code>LogicalType</code> fields in the file metadata and are
-documented in <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>LogicalTypes.md</a></p></div><div class=td-content style=page-break-before:always><h1 id=pg-0d8efcf7d4835e2ce724ae57b02d4318>5 - Nested Encoding</h1><p>To encode nested columns, Parquet uses the Dremel encoding with definition and
+example, strings are stored with the primitive type <code>BYTE_ARRAY</code> with a <code>STRING</code>
+annotation.</p><p>This file contains the specification for all logical types.</p><h3 id=metadata>Metadata</h3><p>The parquet format’s <code>LogicalType</code> stores the type annotation. The annotation
+may require additional metadata fields, as well as rules for those fields.</p><p>There is an older representation of the logical type annotations called <code>ConvertedType</code>.
+To support backward compatibility with old files, readers should interpret <code>LogicalTypes</code>
+in the same way as <code>ConvertedType</code>, and writers should populate <code>ConvertedType</code> in the metadata
+according to well defined conversion rules.</p><h3 id=compatibility>Compatibility</h3><p>The Thrift definition of the metadata has two fields for logical types: <code>ConvertedType</code> and <code>LogicalType</code>.
+<code>ConvertedType</code> is an enum of all available annotations. Since Thrift enums can’t have additional type parameters,
+it is cumbersome to define additional type parameters, like decimal scale and precision
+(which are additional 32 bit integer fields on SchemaElement, and are relevant only for decimals) or time unit
+and UTC adjustment flag for Timestamp types. To overcome this problem, a new logical type representation was introduced into
+the metadata to replace <code>ConvertedType</code>: <code>LogicalType</code>. The new representation is a union of structs of logical types,
+this way allowing more flexible API, logical types can have type parameters.</p><p><code>ConvertedType</code> is deprecated. However, to maintain compatibility with old writers,
+Parquet readers should be able to read and interpret <code>ConvertedType</code> annotations
+in case <code>LogicalType</code> annotations are not present. Parquet writers must always write
+<code>LogicalType</code> annotations where applicable, but must also write the corresponding
+<code>ConvertedType</code> annotations (if any) to maintain compatibility with old readers.</p><p>Compatibility considerations are mentioned for each annotation in the corresponding section.</p><h2 id=string-types>String Types</h2><h3 id=string>STRING</h3><p><code>STRING</code> may only be used to annotate the <code>BYTE_ARRAY</code> primitive type and indicates
+that the byte array should be interpreted as a UTF-8 encoded character string.</p><p>The sort order used for <code>STRING</code> strings is unsigned byte-wise comparison.</p><p><em>Compatibility</em></p><p><code>STRING</code> corresponds to <code>UTF8</code> ConvertedType.</p><h3 id=enum>ENUM</h3><p><code>ENUM</code> annotates the <code>BYTE_ARRAY</code> primitive type and indicates that the value
+was converted from an enumerated type in another data model (e.g. Thrift, Avro, Protobuf).
+Applications using a data model lacking a native enum type should interpret <code>ENUM</code>
+annotated field as a UTF-8 encoded string.</p><p>The sort order used for <code>ENUM</code> values is unsigned byte-wise comparison.</p><h3 id=uuid>UUID</h3><p><code>UUID</code> annotates a 16-byte <code>FIXED_LEN_BYTE_ARRAY</code> primitive type. The value is
+encoded using big-endian, so that <code>00112233-4455-6677-8899-aabbccddeeff</code> is encoded
+as the bytes <code>00 11 22 33 44 55 66 77 88 99 aa bb cc dd ee ff</code>
+(This example is from <a href=https://en.wikipedia.org/wiki/Universally_unique_identifier>wikipedia’s UUID page</a>).</p><p>The sort order used for <code>UUID</code> values is unsigned byte-wise comparison.</p><h2 id=numeric-types>Numeric Types</h2><h3 id=signed-integers>Signed Integers</h3><p><code>INT</code> annotation can be used to specify the maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+For signed integers, the second parameter should be <code>true</code>,
+for example, a signed integer with bit width of 8 is defined as <code>INT(8, true)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, true)</code>, <code>INT(16, true)</code>, and <code>INT(32, true)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, true)</code> must annotate an <code>int64</code> primitive type. <code>INT(32, true)</code> and <code>INT(64, true)</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p>The sort order used for signed integer types is signed.</p><h3 id=unsigned-integers>Unsigned Integers</h3><p><code>INT</code> annotation can be used to specify unsigned integer types,
+along with a maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+In case of unsigned integers, the second parameter should be <code>false</code>,
+for example, an unsigned integer with bit width of 8 is defined as <code>INT(8, false)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, false)</code>, <code>INT(16, false)</code>, and <code>INT(32, false)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, false)</code> must annotate an <code>int64</code> primitive type.</p><p>The sort order used for unsigned integer types is unsigned.</p><h3 id=deprecated-integer-convertedtype>Deprecated integer ConvertedType</h3><p><code>INT_8</code>, <code>INT_16</code>, <code>INT_32</code>, and <code>INT_64</code> annotations can be also used to specify
+signed integers with 8, 16, 32, or 64 bit width.</p><p><code>INT_8</code>, <code>INT_16</code>, and <code>INT_32</code> must annotate an <code>int32</code> primitive type and
+<code>INT_64</code> must annotate an <code>int64</code> primitive type. <code>INT_32</code> and <code>INT_64</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p><code>UINT_8</code>, <code>UINT_16</code>, <code>UINT_32</code>, and <code>UINT_64</code> annotations can be also used to specify
+unsigned integers with 8, 16, 32, or 64 bit width.</p><p><code>UINT_8</code>, <code>UINT_16</code>, and <code>UINT_32</code> must annotate an <code>int32</code> primitive type and
+<code>UINT_64</code> must annotate an <code>int64</code> primitive type.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>INT_8</td><td>IntType (bitWidth = 8, isSigned = true)</td></tr><tr><td>INT_16</td><td>IntType (bitWidth = 16, isSigned = true)</td></tr><tr><td>INT_32</td><td>IntType (bitWidth = 32, isSigned = true)</td></tr><tr><td>INT_64</td><td>IntType (bitWidth = 64, isSigned = true)</td></tr><tr><td [...]
+the form <code>unscaledValue * 10^(-scale)</code>.</p><p>The primitive type stores an unscaled integer value. For <code>BYTE_ARRAY</code> and
+<code>FIXED_LEN_BYTE_ARRAY</code>, the unscaled number must be encoded as two’s complement using
+big-endian byte order (the most significant byte is the zeroth element). The
+scale stores the number of digits of that value that are to the right of the
+decimal point, and the precision stores the maximum number of digits supported
+in the unscaled value.</p><p>If not specified, the scale is 0. Scale must be zero or a positive integer less
+than or equal to the precision. Precision is required and must be a non-zero positive
+integer. A precision too large for the underlying type (see below) is an error.</p><p><code>DECIMAL</code> can be used to annotate the following types:</p><ul><li><code>int32</code>: for 1 <= precision <= 9</li><li><code>int64</code>: for 1 <= precision <= 18; precision < 10 will produce a
+warning</li><li><code>fixed_len_byte_array</code>: precision is limited by the array size. Length <code>n</code>
+can store <= <code>floor(log_10(2^(8*n - 1) - 1))</code> base-10 digits</li><li><code>byte_array</code>: <code>precision</code> is not limited, but is required. The minimum number of
+bytes to store the unscaled value should be used.</li></ul><p>The sort order used for <code>DECIMAL</code> values is signed comparison of the represented
+value.</p><p>If the column uses <code>int32</code> or <code>int64</code> physical types, then signed comparison of
+the integer values produces the correct ordering. If the physical type is
+fixed, then the correct ordering can be produced by flipping the
+most-significant bit in the first byte and then using unsigned byte-wise
+comparison.</p><p><em>Compatibility</em></p><p>To support compatibility with older readers, implementations of parquet-format should
+write <code>DecimalType</code> precision and scale into the corresponding SchemaElement field in metadata.</p><h3 id=float16>FLOAT16</h3><p>The <code>FLOAT16</code> annotation represents half-precision floating-point numbers in the 2-byte IEEE little-endian format.</p><p>Used in contexts where precision is traded off for smaller footprint and potentially better performance.</p><p>The primitive type is a 2-byte <code>FIXED_LEN_BYTE_ARRAY</code>.</p><p>The sort order for <code>FLOAT16</cod [...]
+annotate an <code>int32</code> that stores the number of days from the Unix epoch, 1
+January 1970.</p><p>The sort order used for <code>DATE</code> is signed.</p><h3 id=time>TIME</h3><p><code>TIME</code> is used for a logical time type without a date with millisecond or microsecond precision.
+The type has two type parameters: UTC adjustment (<code>true</code> or <code>false</code>)
+and unit (<code>MILLIS</code> or <code>MICROS</code>, <code>NANOS</code>).</p><p><code>TIME</code> with unit <code>MILLIS</code> is used for millisecond precision.
+It must annotate an <code>int32</code> that stores the number of
+milliseconds after midnight.</p><p><code>TIME</code> with unit <code>MICROS</code> is used for microsecond precision.
+It must annotate an <code>int64</code> that stores the number of
+microseconds after midnight.</p><p><code>TIME</code> with unit <code>NANOS</code> is used for nanosecond precision.
+It must annotate an <code>int64</code> that stores the number of
+nanoseconds after midnight.</p><p>The sort order used for <code>TIME</code> is signed.</p><h4 id=deprecated-time-convertedtype>Deprecated time ConvertedType</h4><p><code>TIME_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int32</code>.</p><p><code>TIME_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MICROS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local time semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local time with legacy <code>TIME_MICROS</code> and <code>TIME_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local time with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIME_MILLIS</td><td>TimeType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIME_MICROS</td><td>TimeType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimeType</td><td rowspan=3>isAdj [...]
+<code>int64</code> number that can be decoded into year, month, day, hour, minute, second
+and subsecond fields using calculations detailed below. Please note that a value
+defined this way does not necessarily correspond to a single instant on the
+time-line and such interpretations are allowed on purpose.</p><p>The <code>TIMESTAMP</code> type has two type parameters:</p><ul><li><code>isAdjustedToUTC</code> must be either <code>true</code> or <code>false</code>.</li><li><code>unit</code> must be one of <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>. This list is subject
+to potential expansion in the future. Upon reading, unknown <code>unit</code>-s must
+be handled as unsupported features (rather than as errors in the data files).</li></ul><h4 id=instant-semantics-timestamps-normalized-to-utc>Instant semantics (timestamps normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=true</code> is defined as the number of
+milliseconds, microseconds or nanoseconds (depending on the <code>unit</code>
+parameter being <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>, respectively) elapsed since the
+Unix epoch, 1970-01-01 00:00:00 UTC. Each such value unambiguously identifies a
+single instant on the time-line.</p><p>For example, in a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>, the
+number 172800000 corresponds to 1970-01-03 00:00:00 UTC, because it is equal to
+2 * 24 * 60 * 60 * 1000, therefore it is exactly two days from the reference
+point, the Unix epoch. In Java, this calculation can be achieved by calling
+<code>Instant.ofEpochMilli(172800000)</code>.</p><p>As a slightly more complicated example, if one wants to store 1970-01-03
+00:00:00 (UTC+01:00) as a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>,
+first the time zone offset has to be dealt with. By normalizing the timestamp to
+UTC, we calculate what time in UTC corresponds to the same instant: 1970-01-02
+23:00:00 UTC. This is 1 day and 23 hours after the epoch, therefore it can be
+encoded as the number (24 + 23) * 60 * 60 * 1000 = 169200000.</p><p>Please note that time zone information gets lost in this process. Upon reading a
+value back, we can only reconstruct the instant, but not the original
+representation. In practice, such timestamps are typically displayed to users in
+their local time zones, therefore they may be displayed differently depending on
+the execution environment.</p><h4 id=local-semantics-timestamps-not-normalized-to-utc>Local semantics (timestamps not normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=false</code> represents year, month, day, hour,
+minute, second and subsecond fields in a local timezone, <em>regardless of what
+specific time zone is considered local</em>. This means that such timestamps should
+always be displayed the same way, regardless of the local time zone in effect.
+On the other hand, without additional information such as an offset or
+time-zone, these values do not identify instants on the time-line unambiguously,
+because the corresponding instants would depend on the local time zone.</p><p>Using a single number to represent a local timestamp is a lot less intuitive
+than for instants. One must use a local timestamp as the reference point and
+calculate the elapsed time between the actual timestamp and the reference point.
+The problem is that the result may depend on the local time zone, for example
+because there may have been a daylight saving time change between the two
+timestamps.</p><p>The solution to this problem is to use a simplification that makes the result
+easy to calculate and independent of the timezone. Treating every day as
+consisting of exactly 86400 seconds and ignoring DST changes altogether allows
+us to unambiguously represent a local timestamp as a difference from a reference
+local timestamp. We define the reference local timestamp to be 1970-01-01
+00:00:00 (note the lack of UTC at the end, as this is not an instant). This way
+the encoding of local timestamp values becomes very similar to the encoding of
+instant values. For example, in a <code>TIMESTAMP(isAdjustedToUTC=false, unit=MILLIS)</code>, the number 172800000 corresponds to 1970-01-03 00:00:00
+(note the lack of UTC at the end), because it is exactly two days from the
+reference point (172800000 = 2 * 24 * 60 * 60 * 1000).</p><p>Another way to get to the same definition is to treat the local timestamp value
+<em>as if</em> it were in UTC and store it as an instant. For example, if we treat the
+local timestamp 1970-01-03 00:00:00 <em>as if</em> it were the instant 1970-01-03
+00:00:00 UTC, we can store it as 172800000. When reading 172800000 back, we can
+reconstruct the instant 1970-01-03 00:00:00 UTC and convert it to a local
+timestamp <em>as if</em> we were in the UTC time zone, resulting in 1970-01-03
+00:00:00. In Java, this can be achieved by calling
+<code>LocalDateTime.ofEpochSecond(172800, 0, ZoneOffset.UTC)</code>.</p><p>Please note that while from a practical point of view this second definition is
+equivalent to the first one, from a theoretical point of view only the first
+definition can be considered correct, the second one just “incidentally” leads
+to the same results. Nevertheless, this second definition is worth mentioning as
+well, because it is relatively widespread and it can lead to confusion,
+especially due to its usage of UTC in the calculations. One can stumble upon
+code, comments and specifications ambiguously stating that a timestamp “is
+stored in UTC”. In some contexts, it means that it is <em>normalized</em> to UTC and
+acts as an instant. In some other contexts though, it means the exact opposite,
+namely that the timestamp is stored <em>as if</em> it were in UTC and acts as a
+local timestamp in reality.</p><h4 id=common-considerations>Common considerations</h4><p>Every possible <code>int64</code> number represents a valid timestamp, but depending on the
+precision, the corresponding year may be outside of the practical everyday
+limits and implementations may choose to only support a limited range.</p><p>On the other hand, not every combination of year, month, day, hour, minute,
+second and subsecond values can be encoded into an <code>int64</code>. Most notably:</p><ul><li>An arbitrary combination of timestamp fields can not be encoded as a single
+number if the values for some of the fields are outside of their normal range
+(where the “normal range” corresponds to everyday usage). For example, neither
+of the following can be represented in a timestamp:<ul><li>hour = -1</li><li>hour = 25</li><li>minute = 61</li><li>month = 13</li><li>day = 29, month = 2, year = any non-leap year</li></ul></li><li>Due to the range of the <code>int64</code> type, timestamps using the <code>NANOS</code> unit
+can only represent values between 1677-09-21 00:12:43 and 2262-04-11 23:47:16.
+Values outside of this range can not be represented with the <code>NANOS</code>
+unit. (Other precisions have similar limits but those are outside of the
+domain for practical everyday usage.)</li></ul><p>The sort order used for <code>TIMESTAMP</code> is signed.</p><h4 id=deprecated-timestamp-convertedtype>Deprecated timestamp ConvertedType</h4><p><code>TIMESTAMP_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p><code>TIMESTAMP_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MICROS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local timestamp semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local timestamps with legacy <code>TIMESTAMP_MICROS</code> and <code>TIMESTAMP_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local timestamps with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIMESTAMP_MILLIS</td><td>TimestampType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIMESTAMP_MICROS</td><td>TimestampType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimestampTyp [...]
+<code>fixed_len_byte_array</code> of length 12. This array stores three little-endian
+unsigned integers that represent durations at different granularities of time.
+The first stores a number in months, the second stores a number in days, and
+the third stores a number in milliseconds. This representation is independent
+of any particular timezone or date.</p><p>Each component in this representation is independent of the others. For
+example, there is no requirement that a large number of days should be
+expressed as a mix of months and days because there is not a constant
+conversion from days to months.</p><p>The sort order used for <code>INTERVAL</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=embedded-types>Embedded Types</h2><p>Embedded types do not have type-specific orderings.</p><h3 id=json>JSON</h3><p><code>JSON</code> is used for an embedded JSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as a UTF-8 encoded character
+string of valid JSON as defined by the <a href=http://json.org/>JSON specification</a></p><p>The sort order used for <code>JSON</code> is unsigned byte-wise comparison.</p><h3 id=bson>BSON</h3><p><code>BSON</code> is used for an embedded BSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as an encoded BSON document as
+defined by the <a href=http://bsonspec.org/spec.html>BSON specification</a>.</p><p>The sort order used for <code>BSON</code> is unsigned byte-wise comparison.</p><h3 id=variant>VARIANT</h3><p><code>VARIANT</code> is used for a Variant value. It must annotate a group. The group must
+contain a field named <code>metadata</code> and a field named <code>value</code>. Both fields must have
+type <code>binary</code>, which is also called <code>BYTE_ARRAY</code> in the Parquet thrift definition.
+The <code>VARIANT</code> annotated group can be used to store either an unshredded Variant
+value, or a shredded Variant value.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</code> must be of type <code>binary</code> (called <code>BYTE_ARRAY</code>
+in the Parquet thrift definition).</li><li>The <code>metadata</code> field is required and must be a valid Variant metadata component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>When present, the <code>value</code> field must be a valid Variant value component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>The <code>value</code> field is required for unshredded Variant values.</li><li>The <code>value</code> field is optional and may be null only when parts of the Variant
+value are shredded according to the <a href=/docs/file-format/types/variantshredding/>Variant shredding specification</a>.</li></ul><p>This is the expected representation of an unshredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_unshredded (VARIANT) {
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_shredded (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><h3 id=geometry>GEOMETRY</h3><p><code>GEOMETRY</code> is used for geospatial features in the Well-Known Binary (WKB) format
+with linear/planar edges interpolation. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has only one type parameter:</p><ul><li><code>crs</code>: An optional string value for CRS. If unset, the CRS defaults to
+<code>"OGC:CRS84"</code>, which means that the geometries must be stored in longitude,
+latitude based on the WGS84 datum.</li></ul><p>The sort order used for <code>GEOMETRY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h3 id=geography>GEOGRAPHY</h3><p><code>GEOGRAPHY</code> is used for geospatial features in the WKB format with an explicit
+(non-linear/non-planar) edges interpolation algorithm. It must annotate a
+<code>BYTE_ARRAY</code> primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has two type parameters:</p><ul><li><code>crs</code>: An optional string value for CRS. It must be a geographic CRS, where
+longitudes are bound by [-180, 180] and latitudes are bound by [-90, 90].
+If unset, the CRS defaults to <code>"OGC:CRS84"</code>.</li><li><code>algorithm</code>: An optional enum value to describes the edge interpolation
+algorithm. Supported values are: <code>SPHERICAL</code>, <code>VINCENTY</code>, <code>THOMAS</code>, <code>ANDOYER</code>,
+<code>KARNEY</code>. If unset, the algorithm defaults to <code>SPHERICAL</code>.</li></ul><p>The sort order used for <code>GEOGRAPHY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=nested-types>Nested Types</h2><p>This section specifies how <code>LIST</code> and <code>MAP</code> can be used to encode nested types
+by adding group levels around repeated fields that are not present in the data.</p><p>This does not affect repeated fields that are not annotated: A repeated field
+that is neither contained by a <code>LIST</code>- or <code>MAP</code>-annotated group nor annotated
+by <code>LIST</code> or <code>MAP</code> should be interpreted as a required list of required
+elements where the element type is the type of the field.</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// List<Integer> (non-null list, non-null elements)
+repeated int32 num;
+
+// List<Tuple<Integer, String>> (non-null list, non-null elements)
+repeated group my_list {
+ required int32 num;
+ optional binary str (STRING);
+}
+</code></pre><p>For all fields in the schema, implementations should use either <code>LIST</code> and
+<code>MAP</code> annotations <em>or</em> unannotated repeated fields, but not both. When using
+the annotations, no unannotated repeated types are allowed.</p><h3 id=lists>Lists</h3><p><code>LIST</code> is used to annotate types that should be interpreted as lists.</p><p><code>LIST</code> must always annotate a 3-level structure:</p><pre tabindex=0><code><list-repetition> group <name> (LIST) {
+ repeated group list {
+ <element-repetition> <element-type> element;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>LIST</code> that contains a
+single field named <code>list</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the list is nullable.</li><li>The middle level, named <code>list</code>, must be a repeated group with a single
+field named <code>element</code>.</li><li>The <code>element</code> field encodes the list’s element type and repetition. Element
+repetition must be <code>required</code> or <code>optional</code>.</li></ul><p>The following examples demonstrate two of the possible lists of string values.</p><pre tabindex=0><code>// List<String> (list non-null, elements nullable)
+required group my_list (LIST) {
+ repeated group list {
+ optional binary element (STRING);
+ }
+}
+
+// List<String> (list nullable, elements non-null)
+optional group my_list (LIST) {
+ repeated group list {
+ required binary element (STRING);
+ }
+}
+</code></pre><p>Element types can be nested structures. For example, a list of lists:</p><pre tabindex=0><code>// List<List<Integer>>
+optional group array_of_arrays (LIST) {
+ repeated group list {
+ required group element (LIST) {
+ repeated group list {
+ required int32 element;
+ }
+ }
+ }
+}
+</code></pre><h4 id=backward-compatibility-rules>Backward-compatibility rules</h4><p>New writer implementations should always produce the 3-level LIST structure shown
+above. However, historically data files have been produced that use different
+structures to represent list-like data, and readers may include compatibility
+measures to interpret them as intended.</p><p>It is required that the repeated group of elements is named <code>list</code> and that
+its element field is named <code>element</code>. However, these names may not be used in
+existing data and should not be enforced as errors when reading. For example,
+the following field schema should produce a nullable list of non-null strings,
+even though the repeated group is named <code>element</code>.</p><pre tabindex=0><code>optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ };
+}
+</code></pre><p>Some existing data does not include the inner element layer, resulting in a
+<code>LIST</code> that annotates a 2-level structure. Unlike the 3-level structure, the
+repetition of a 2-level structure can be <code>optional</code>, <code>required</code>, or <code>repeated</code>.
+When it is <code>repeated</code>, the <code>LIST</code>-annotated 2-level structure can only serve as
+an element within another <code>LIST</code>-annotated 2-level structure.</p><p>For backward-compatibility, the type of elements in <code>LIST</code>-annotated structures
+should always be determined by the following rules:</p><ol><li>If the repeated field is not a group, then its type is the element type and
+elements are required.</li><li>If the repeated field is a group with multiple fields, then its type is the
+element type and elements are required.</li><li>If the repeated field is a group with one field with <code>repeated</code> repetition,
+then its type is the element type and elements are required.</li><li>If the repeated field is a group with one field and is named either <code>array</code>
+or uses the <code>LIST</code>-annotated group’s name with <code>_tuple</code> appended then the
+repeated type is the element type and elements are required.</li><li>Otherwise, the repeated field’s type is the element type with the repeated
+field’s repetition.</li></ol><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// Rule 1: List<Integer> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated int32 element;
+}
+
+// Rule 2: List<Tuple<String, Integer>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ required int32 num;
+ };
+}
+
+// Rule 3: List<List<Integer>> (nullable outer list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array (LIST) {
+ repeated int32 array;
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array {
+ required binary str (STRING);
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group my_list_tuple {
+ required binary str (STRING);
+ };
+}
+
+// Rule 5: List<String> (nullable list, nullable elements)
+optional group my_list (LIST) {
+ repeated group element {
+ optional binary str (STRING);
+ };
+}
+</code></pre><h3 id=maps>Maps</h3><p><code>MAP</code> is used to annotate types that should be interpreted as a map from keys
+to values. <code>MAP</code> must annotate a 3-level structure:</p><pre tabindex=0><code><map-repetition> group <name> (MAP) {
+ repeated group key_value {
+ required <key-type> key;
+ <value-repetition> <value-type> value;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>MAP</code> that contains a
+single field named <code>key_value</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the map is nullable.</li><li>The middle level, named <code>key_value</code>, must be a repeated group with a <code>key</code>
+field for map keys and, optionally, a <code>value</code> field for map values. It must
+not contain any other values.</li><li>The <code>key</code> field encodes the map’s key type. This field must have
+repetition <code>required</code> and must always be present. It must always be the first
+field of the repeated <code>key_value</code> group.</li><li>The <code>value</code> field encodes the map’s value type and repetition. This field can
+be <code>required</code>, <code>optional</code>, or omitted. It must always be the second field of
+the repeated <code>key_value</code> group if present. In case of not present, it can be
+represented as a map with all null values or as a set of keys.</li></ul><p>The following example demonstrates the type for a non-null map from strings to
+nullable integers:</p><pre tabindex=0><code>// Map<String, Integer>
+required group my_map (MAP) {
+ repeated group key_value {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><p>If there are multiple key-value pairs for the same key, then the final value
+for that key must be the last value. Other values may be ignored or may be
+added with replacement to the map container in the order that they are encoded.
+The <code>MAP</code> annotation should not be used to encode multi-maps using duplicate
+keys.</p><h4 id=backward-compatibility-rules-1>Backward-compatibility rules</h4><p>It is required that the repeated group of key-value pairs is named <code>key_value</code>
+and that its fields are named <code>key</code> and <code>value</code>. However, these names may not
+be used in existing data and should not be enforced as errors when reading.
+(<code>key</code> and <code>value</code> can be identified by their position in case of misnaming.)</p><p>Some existing data incorrectly used <code>MAP_KEY_VALUE</code> in place of <code>MAP</code>. For
+backward-compatibility, a group annotated with <code>MAP_KEY_VALUE</code> that is not
+contained by a <code>MAP</code>-annotated group should be handled as a <code>MAP</code>-annotated
+group.</p><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>// Map<String, Integer> (nullable map, non-null values)
+optional group my_map (MAP) {
+ repeated group map {
+ required binary str (STRING);
+ required int32 num;
+ }
+}
+
+// Map<String, Integer> (nullable map, nullable values)
+optional group my_map (MAP_KEY_VALUE) {
+ repeated group map {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><h2 id=unknown-always-null>UNKNOWN (always null)</h2><p>Sometimes, when discovering the schema of existing data, values are always null
+and there’s no type information.
+The <code>UNKNOWN</code> type can be used to annotate a column that is always null.
+(Similar to Null type in Avro and Arrow)</p></div><div class=td-content style=page-break-before:always><h1 id=pg-29a57736b5ceb62cc6d5a0ba8ba429b0>5.3 -</h1><h1 id=variant-shredding>Variant Shredding</h1><p>The Variant type is designed to store and process semi-structured data efficiently, even with heterogeneous values.
+Query engines encode each Variant value in a self-describing format, and store it as a group containing <code>value</code> and <code>metadata</code> binary fields in Parquet.
+Since data is often partially homogeneous, it can be beneficial to extract certain fields into separate Parquet columns to further improve performance.
+This process is called <strong>shredding</strong>.</p><p>Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections.</p><p>For example, the query <code>SELECT variant_get(event, '$.event_ts', 'timestamp') FROM tbl</code> only needs to load field <code>event_ts</code>, and if that column is shredded, it can be read by columnar projection without reading or deserializing the rest of the < [...]
+Similarly, for the query <code>SELECT * FROM tbl WHERE variant_get(event, '$.event_type', 'string') = 'signup'</code>, the <code>event_type</code> shredded column metadata can be used for skipping and to lazily load the rest of the Variant.</p><h2 id=variant-metadata>Variant Metadata</h2><p>Variant metadata is stored in the top-level Variant group in a binary <code>metadata</code> column regardless of whether the Variant value is shredded.</p><p>All <code>value</code> columns within the [...]
+All field names of a Variant, whether shredded or not, must be present in the metadata.</p><h2 id=value-shredding>Value Shredding</h2><p>Variant values are stored in Parquet fields named <code>value</code>.
+Each <code>value</code> field may have an associated shredded field named <code>typed_value</code> that stores the value when it matches a specific type.
+When <code>typed_value</code> is present, readers <strong>must</strong> reconstruct shredded values according to this specification.</p><p>For example, a Variant field, <code>measurement</code> may be shredded as long values by adding <code>typed_value</code> with type <code>int64</code>:</p><pre tabindex=0><code>required group measurement (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The Parquet columns used to store variant metadata and values must be accessed by name, not by position.</p><p>The series of measurements <code>34, null, "n/a", 100</code> would be stored as:</p><table><thead><tr><th>Value</th><th><code>metadata</code></th><th><code>value</code></th><th><code>typed_value</code></th></tr></thead><tbody><tr><td>34</td><td><code>01 00</code> v1/empty</td><td>null</td><td><code>34</code></td></tr><tr><td>null</td><td><code>01 00</code> v1/emp [...]
+Values in the two fields must be interpreted according to the following table:</p><table><thead><tr><th><code>value</code></th><th><code>typed_value</code></th><th>Meaning</th></tr></thead><tbody><tr><td>null</td><td>null</td><td>The value is missing; only valid for shredded object fields</td></tr><tr><td>non-null</td><td>null</td><td>The value is present and may be any type, including null</td></tr><tr><td>null</td><td>non-null</td><td>The value is present and is the shredded type</td>< [...]
+Writers must not produce data where both <code>value</code> and <code>typed_value</code> are non-null, unless the Variant value is an object.</p><p>If a Variant is missing in a context where a value is required, readers must return a Variant null (<code>00</code>): basic type 0 (primitive) and physical type 0 (null).
+For example, if a Variant is required (like <code>measurement</code> above) and both <code>value</code> and <code>typed_value</code> are null, the returned <code>value</code> must be <code>00</code> (Variant null).</p><h3 id=shredded-value-types>Shredded Value Types</h3><p>Shredded values must use the following Parquet types:</p><table><thead><tr><th>Variant Type</th><th>Parquet Physical Type</th><th>Parquet Logical Type</th></tr></thead><tbody><tr><td>boolean</td><td>BOOLEAN</td><td></t [...]
+If the value is an array, <code>value</code> must be null.</p><p>The list <code>element</code> must be a required group.
+The <code>element</code> group can contain <code>value</code> and <code>typed_value</code> fields.
+The element’s <code>value</code> field stores the element as Variant-encoded <code>binary</code> when the <code>typed_value</code> is not present or cannot represent it.
+The <code>typed_value</code> field may be omitted when not shredding elements as a specific type.
+The <code>value</code> field may be omitted when shredding elements as a specific type.
+However, at least one of the two fields must be present.</p><p>For example, a <code>tags</code> Variant may be shredded as a list of strings using the following definition:</p><pre tabindex=0><code>optional group tags (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value (LIST) { # must be optional to allow a null list
+ repeated group list {
+ required group element { # shredded element
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+}
+</code></pre><p>All elements of an array must be present (not missing) because the <code>array</code> Variant encoding does not allow missing elements.
+That is, either <code>typed_value</code> or <code>value</code> (but not both) must be non-null.
+Null elements must be encoded in <code>value</code> as Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>The series of <code>tags</code> arrays <code>["comedy", "drama"], ["horror", null], ["comedy", "drama", "romance"], null</code> would be stored as:</p><table><thead><tr><th>Array</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value...value</code></th><th><code>typed_value...typed_value</code></th></tr></thead><tbody><tr><td><code [...]
+If the value is not an object, <code>typed_value</code> must be null.
+Readers can assume that a value is not an object if <code>typed_value</code> is null and that <code>typed_value</code> field values are correct; that is, readers do not need to read the <code>value</code> column if <code>typed_value</code> fields satisfy the required fields.</p><p>Each shredded field in the <code>typed_value</code> group is represented as a required group that contains optional <code>value</code> and <code>typed_value</code> fields.
+The <code>value</code> field stores the value as Variant-encoded <code>binary</code> when the <code>typed_value</code> cannot represent the field.
+This layout enables readers to skip data based on the field statistics for <code>value</code> and <code>typed_value</code>.
+The <code>typed_value</code> field may be omitted when not shredding fields as a specific type.</p><p>The <code>value</code> column of a partially shredded object must never contain fields represented by the Parquet columns in <code>typed_value</code> (shredded fields).
+Readers may always assume that data is written correctly and that shredded fields in <code>typed_value</code> are not present in <code>value</code>.
+As a result, reads when a field is defined in both <code>value</code> and a <code>typed_value</code> shredded field may be inconsistent.</p><p>For example, a Variant <code>event</code> field may shred <code>event_type</code> (<code>string</code>) and <code>event_ts</code> (<code>timestamp</code>) columns using the following definition:</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value; # a variant, expected to be an object
+ optional group typed_value { # shredded fields for the variant object
+ required group event_type { # shredded field for event_type
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts { # shredded field for event_ts
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ }
+}
+</code></pre><p>The group for each named field must use repetition level <code>required</code>.</p><p>A field’s <code>value</code> and <code>typed_value</code> are set to null (missing) to indicate that the field does not exist in the variant.
+To encode a field that is present with a null value, the <code>value</code> must contain a Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>When both <code>value</code> and <code>typed_value</code> for a field are non-null, engines should fail.
+If engines choose to read in such cases, then the <code>typed_value</code> column must be used.
+Readers may always assume that data is written correctly and that only <code>value</code> or <code>typed_value</code> is defined.
+As a result, reads when both <code>value</code> and <code>typed_value</code> are defined may be inconsistent with optimized reads that require only one of the columns.</p><p>The table below shows how the series of objects in the first column would be stored:</p><table><thead><tr><th>Event object</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value.event_type.value</code></th><th><code>typed_value.event_type.typed_value</code></th><th><code>typed_value.eve [...]
+Readers must return an object when <code>typed_value</code> is non-null containing the shredded fields.</p><h2 id=nesting>Nesting</h2><p>The <code>typed_value</code> associated with any Variant <code>value</code> field can be any shredded type, as shown in the sections above.</p><p>For example, the <code>event</code> object above may also shred sub-fields as object (<code>location</code>) or array (<code>tags</code>).</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value {
+ required group event_type {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts {
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ required group location {
+ optional binary value;
+ optional group typed_value {
+ required group latitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ required group longitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ }
+ }
+ required group tags {
+ optional binary value;
+ optional group typed_value (LIST) {
+ repeated group list {
+ required group element {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+ }
+ }
+}
+</code></pre><h1 id=data-skipping>Data Skipping</h1><p>Statistics for <code>typed_value</code> columns can be used for file, row group, or page skipping when <code>value</code> is always null (missing).</p><p>When the corresponding <code>value</code> column is all nulls, all values must be the shredded <code>typed_value</code> field’s type.
+Because the type is known, comparisons with values of that type are valid.
+<code>IS NULL</code>/<code>IS NOT NULL</code> and <code>IS NAN</code>/<code>IS NOT NAN</code> filter results are also valid.</p><p>Comparisons with values of other types are not necessarily valid and data should not be skipped.</p><p>Casting behavior for Variant is delegated to processing engines.
+For example, the interpretation of a string as a timestamp may depend on the engine’s SQL session time zone.</p><h2 id=reconstructing-a-shredded-variant>Reconstructing a Shredded Variant</h2><p>It is possible to recover an unshredded Variant value using a recursive algorithm, where the initial call is to <code>construct_variant</code> with the top-level Variant group fields.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-siz [...]
+</span></span><span style=display:flex><span> <span style=color:#4e9a06>"""Constructs a Variant from value and typed_value"""</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>typed_value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>dict</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#000>object_fields</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#000>name</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> < [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>name</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#000>typed_value</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a partially shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>),</span> <span style=color:#4e9a06>"partially shredded value must be an object"</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>typed_value</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>keys</span><span style=color:#000;font-weight:700>()</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>isdisjoint</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#ce5c00;font [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># union the shredded fields and non-shredded fields</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>union</span><span style=color:#000;f [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded array</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded array must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#000>elements</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>[</span>
+</span></span><span style=display:flex><span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span> [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000>elem</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantArray</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elements</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded primitive</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded primitive must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># value is missing</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#204a87;font-weight:700>None</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>def</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>Any</span><span style=color:#000;font-weight:700>):</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>int</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantInteger</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>str</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantString</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>...</span>
+</span></span></code></pre></div><h2 id=backward-and-forward-compatibility>Backward and forward compatibility</h2><p>Shredding is an optional feature of Variant, and readers must continue to be able to read a group containing only <code>value</code> and <code>metadata</code> fields.</p><p>Engines that do not write shredded values must be able to read shredded values according to this spec or must fail.</p><p>Different files may contain conflicting shredding schemas.
+That is, files may contain different <code>typed_value</code> columns for the same Variant with incompatible types.
+It may not be possible to infer or specify a single shredded schema that would allow all Parquet files for a table to be read without reconstructing the value as a Variant.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-ad7f0d170ee709f2e58f9e27d7567421>5.4 -</h1><h1 id=variant-binary-encoding>Variant Binary Encoding</h1><p>A Variant represents a type that contains one of:</p><ul><li>Primitive: A type and corresponding value (e.g. INT, STRING)</li><li>Array: An or [...]
+These represent a commonly supported subset of the <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>logical types</a> allowed by the Parquet format.</p><p>The Variant Binary Encoding allows representation of semi-structured data (e.g. JSON) in a form that can be efficiently queried by path.
+The design is intended to allow efficient access to nested data even in the presence of very wide or deep structures.</p><p>Another motivation for the representation is that (aside from metadata) each nested Variant value is contiguous and self-contained.
+For example, in a Variant containing an Array of Variant values, the representation of an inner Variant value, when paired with the metadata of the full variant, is itself a valid Variant.</p><p>This document describes the Variant Binary Encoding scheme.
+Variant fields can also be <em>shredded</em>.
+Shredding refers to extracting some elements of the variant into separate columns for more efficient extraction/filter pushdown.
+The <a href=/docs/file-format/types/variantshredding/>Variant Shredding specification</a> describes the details of shredding Variant values as typed Parquet columns.</p><h2 id=variant-in-parquet>Variant in Parquet</h2><p>A Variant value in Parquet is represented by a group with 2 fields, named <code>value</code> and <code>metadata</code>.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</c [...]
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group shredded_variant_name (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The <code>VARIANT</code> annotation places no additional restrictions on the repetition of Variant groups, but repetition may be restricted by containing types (such as <code>MAP</code> and <code>LIST</code>).
+The Variant group name is the name of the Variant column.</p><h2 id=metadata-encoding>Metadata encoding</h2><p>The encoded metadata always starts with a header byte.</p><pre tabindex=0><code> 7 6 5 4 3 0
+ +-------+---+---+---------------+
+header | | | | version |
+ +-------+---+---+---------------+
+ ^ ^
+ | +-- sorted_strings
+ +-- offset_size_minus_one
+</code></pre><p>The <code>version</code> is a 4-bit value that must always contain the value <code>1</code>.
+<code>sorted_strings</code> is a 1-bit value indicating whether dictionary strings are sorted and unique.
+<code>offset_size_minus_one</code> is a 2-bit value providing the number of bytes per dictionary size and offset field.
+The actual number of bytes, <code>offset_size</code>, is <code>offset_size_minus_one + 1</code>.</p><p>The entire metadata is encoded as the following diagram shows:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+metadata | header |
+ +-----------------------+
+ | |
+ : dictionary_size : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | | (`dictionary_size + 1` offsets)
+ +-----------------------+
+ | |
+ : bytes :
+ | |
+ +-----------------------+
+</code></pre><p>The metadata is encoded first with the <code>header</code> byte, then <code>dictionary_size</code> which is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the number of string values in the dictionary.
+Next, is an <code>offset</code> list, which contains <code>dictionary_size + 1</code> values.
+Each <code>offset</code> is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the starting byte offset of the i-th string in <code>bytes</code>.
+The first <code>offset</code> value will always be <code>0</code>, and the last <code>offset</code> value will always be the total length of <code>bytes</code>.
+The last part of the metadata is <code>bytes</code>, which stores all the string values in the dictionary.
+All string values must be UTF-8 encoded strings.</p><h3 id=metadata-encoding-grammar>Metadata encoding grammar</h3><p>The grammar for encoded metadata is as follows</p><pre tabindex=0><code>metadata: <header> <dictionary_size> <dictionary>
+header: 1 byte (<version> | <sorted_strings> << 4 | (<offset_size_minus_one> << 6))
+version: a 4-bit version ID. Currently, must always contain the value 1
+sorted_strings: a 1-bit value indicating whether metadata strings are sorted
+offset_size_minus_one: 2-bit value providing the number of bytes per dictionary size and offset field.
+dictionary_size: `offset_size` bytes. unsigned little-endian value indicating the number of strings in the dictionary
+dictionary: <offset>* <bytes>
+offset: `offset_size` bytes. unsigned little-endian value indicating the starting position of the ith string in `bytes`. The list should contain `dictionary_size + 1` values, where the last value is the total length of `bytes`.
+bytes: UTF-8 encoded dictionary string values
+</code></pre><p>Notes:</p><ul><li>Offsets are relative to the start of the <code>bytes</code> array.</li><li>The length of the ith string can be computed as <code>offset[i+1] - offset[i]</code>.</li><li>The offset of the first string is always equal to 0 and is therefore redundant. It is included in the spec to simplify in-memory-processing.</li><li><code>offset_size_minus_one</code> indicates the number of bytes per <code>dictionary_size</code> and <code>offset</code> entry. I.e. a valu [...]
+ +------------------------------------+------------+
+value | value_header | basic_type | <-- value_metadata
+ +------------------------------------+------------+
+ | |
+ : value_data : <-- 0 or more bytes
+ | |
+ +-------------------------------------------------+
+</code></pre><h3 id=basic-type>Basic Type</h3><p>The <code>basic_type</code> is 2-bit value that represents which basic type the Variant value is.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>basic types table</a> shows what each value represents.</p><h3 id=value-header>Value Header</h3><p>The <code>value_header</code> is a 6-bit value that contains more information about the type, and the format depends on the <code>basic_type</code>.</p><h4 id=value-header-for-primitive-type-basic_type0>Value Header for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>v [...]
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows what each value represents.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | primitive_header |
+ +-----------------------+
+</code></pre><h4 id=value-header-for-short-string-basic_type1>Value Header for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_header</code> is a 6-bit <code>short_string_header</code>.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | short_string_header |
+ +-----------------------+
+</code></pre><p>The <code>short_string_header</code> value is the length of the string.</p><h4 id=value-header-for-object-basic_type2>Value Header for Object (<code>basic_type</code>=2)</h4><p>When <code>basic_type</code> is <code>2</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, <code>field_id_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 4 3 2 1 0
+ +---+---+-------+-------+
+value_header | | | | |
+ +---+---+-------+-------+
+ ^ ^ ^
+ | | +-- field_offset_size_minus_one
+ | +-- field_id_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> and <code>field_id_size_minus_one</code> are 2-bit values that represent the number of bytes used to encode the field offsets and field ids.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code> and <code>field_id_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h4 id=value-header-for-array-basic_type3>Value Header for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 3 2 1 0
+ +-----------+---+-------+
+value_header | | | |
+ +-----------+---+-------+
+ ^ ^
+ | +-- field_offset_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> is a 2-bit value that represents the number of bytes used to encode the field offset.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h3 id=value-data>Value Data</h3><p>The <code>value_data</code> encoding format depends on the type specified by <code>value_metadata</code>.
+For some types, the <code>value_data</code> will be 0-bytes.</p><h4 id=value-data-for-primitive-type-basic_type0>Value Data for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>value_data</code> depends on the <code>primitive_header</code> value.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows the encoding format for each primitive type.</p><h4 id=value-data-for-short-string-basic_type1>Value Data for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_data</code> is the sequence of UTF-8 encoded bytes that represents the string.</p><h4 id=value-data-for-object-basic_type2>Value Data for Object (<code>basic_type</code>=2)</ [...]
+The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+object value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | | (`num_elements` field_ids)
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An object <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the object.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a list of <code>field_id</code> values.
+There are <code>num_elements</code> number of entries and each <code>field_id</code> is an unsigned little-endian value of <code>field_id_size</code> bytes.
+A <code>field_id</code> is an index into the dictionary in the metadata.
+The <code>field_id</code> list is followed by a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the end of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th key-value pair of the object, the key is the metadata dictionary entry indexed by the i-th <code>field_id</code>, and the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><p>The field ids and field offsets must be in lexicographical order of the corresponding field names in the metadata dictionary.
+However, the actual <code>value</code> entries do not need to be in any particular order.
+This implies that the <code>field_offset</code> values may not be monotonically increasing.
+For example, for the following object:</p><pre tabindex=0><code>{
+ "c": 3,
+ "b": 2,
+ "a": 1
+}
+</code></pre><p>The <code>field_id</code> list must be <code>[<id for key "a">, <id for key "b">, <id for key "c">]</code>, in lexicographical order.
+The <code>field_offset</code> list must be <code>[<offset for value 1>, <offset for value 2>, <offset for value 3>, <last offset>]</code>.
+The <code>value</code> list can be in any order.</p><h4 id=value-data-for-array-basic_type3>Value Data for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_data</code> encodes an array. The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+array value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An array <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the array.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the last byte of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th array entry, the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><h3 id=value-encoding-grammar>Value encoding grammar</h3><p>The grammar for an encoded value is:</p><pre tabindex=0><code>value: <value_metadata> <value_data>?
+value_metadata: 1 byte (<basic_type> | (<value_header> << 2))
+basic_type: ID from Basic Type table. <value_header> must be a corresponding variation
+value_header: <primitive_header> | <short_string_header> | <object_header> | <array_header>
+primitive_header: ID from Primitive Type table. <val> must be a corresponding variation of <primitive_val>
+short_string_header: unsigned string length in bytes from 0 to 63
+object_header: (is_large << 4 | field_id_size_minus_one << 2 | field_offset_size_minus_one)
+array_header: (is_large << 2 | field_offset_size_minus_one)
+value_data: <primitive_val> | <short_string_val> | <object_val> | <array_val>
+primitive_val: see table for binary representation
+short_string_val: UTF-8 encoded bytes
+object_val: <num_elements> <field_id>* <field_offset>* <fields>
+array_val: <num_elements> <field_offset>* <fields>
+num_elements: a 1 or 4 byte unsigned little-endian value (depending on is_large in <object_header>/<array_header>)
+field_id: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_id_size_minus_one in <object_header>), indexing into the dictionary
+field_offset: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_offset_size_minus_one in <object_header>/<array_header>), providing the offset in bytes within fields
+fields: <value>*
+</code></pre><p>Each <code>value_data</code> must correspond to the type defined by <code>value_metadata</code>. Boolean and null types do not have a corresponding <code>value_data</code>, since their type defines their value.</p><p>Each <code>array_val</code> and <code>object_val</code> must contain exactly <code>num_elements + 1</code> values for <code>field_offset</code>.
+The last entry is the offset that is one byte past the last field (i.e. the total size of all fields in bytes).
+All offsets are relative to the first byte of the first field in the object/array.</p><p><code>field_id_size_minus_one</code> and <code>field_offset_size_minus_one</code> indicate the number of bytes per field ID/offset.
+For example, a value of 0 indicates 1-byte IDs, 1 indicates 2-byte IDs, 2 indicates 3 byte IDs and 3 indicates 4-byte IDs.
+The <code>is_large</code> flag for arrays and objects is used to indicate whether the number of elements is indicated using a one or four byte value.
+When more than 255 elements are present, <code>is_large</code> must be set to true.
+It is valid for an implementation to use a larger value than necessary for any of these fields (e.g. <code>is_large</code> may be true for an object with less than 256 elements).</p><p>The “short string” basic type may be used as an optimization to fold string length into the type byte for strings less than 64 bytes.
+It is semantically identical to the “string” primitive type.</p><p>The Decimal type contains a scale, but no precision. The implied precision of a decimal value is <code>floor(log_10(val)) + 1</code>.</p><h2 id=encoding-types>Encoding types</h2><p><em>Variant basic types</em></p><table><thead><tr><th>Basic Type</th><th>ID</th><th>Description</th></tr></thead><tbody><tr><td>Primitive</td><td><code>0</code></td><td>One of the primitive types</td></tr><tr><td>Short string</td><t [...]
+For example, a user expression operating on a string value containing “hello” should behave the same, whether it is encoded with the short string optimization, or long string encoding.
+Similarly, user expressions operating on an <em>int8</em> value of 1 should behave the same as a decimal16 with scale 2 and unscaled value 100.</p><p><em>Decimal table</em></p><table><thead><tr><th>Decimal Precision</th><th>Decimal value type</th><th>Variant Physical Type</th></tr></thead><tbody><tr><td>1 <= precision <= 9</td><td>int32</td><td>decimal4</td></tr><tr><td>10 <= precision <= 18</td><td>int64</td><td>decimal8</td></tr><tr><td>19 <= precision <= 38</td><td>i [...]
+This includes the dictionary key string values, the “short string” values, and the “long string” values.</p><h2 id=object-field-id-order-and-uniqueness>Object field ID order and uniqueness</h2><p>For objects, field IDs and offsets must be listed in the order of the corresponding field names, sorted lexicographically (using unsigned byte ordering for UTF-8).
+Note that the field values themselves are not required to follow this order.
+As a result, offsets will not necessarily be listed in ascending order.
+The field values are not required to be in the same order as the field IDs, to enable flexibility when constructing Variant values.</p><p>An implementation may rely on this field ID order in searching for field names.
+E.g. a binary search on field IDs (combined with metadata lookups) may be used to find a field with a given name.</p><p>Field names are case-sensitive.
+Field names are required to be unique for each object.
+It is an error for an object to contain two fields with the same name, whether or not they have distinct dictionary IDs.</p><h2 id=versions-and-extensions>Versions and extensions</h2><p>An implementation is not expected to parse a Variant value whose metadata version is higher than the version supported by the implementation.
+However, new types may be added to the specification without incrementing the version ID.
+In such a situation, an implementation should be able to read the rest of the Variant value if desired.</p><h2 id=shredding>Shredding</h2><p>A single Variant object may have poor read performance when only a small subset of fields are needed.
+A better approach is to create separate columns for individual fields, referred to as shredding or subcolumnarization.
+<a href=/docs/file-format/types/variantshredding/>VariantShredding.md</a> describes the Variant shredding specification in Parquet.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-0d8efcf7d4835e2ce724ae57b02d4318>6 - Nested Encoding</h1><p>To encode nested columns, Parquet uses the Dremel encoding with definition and
repetition levels. Definition levels specify how many optional fields in the
path for the column are defined. Repetition levels specify at what repeated field
in the path has the value repeated. The max definition and repetition levels can
be computed from the schema (i.e. how much nesting there is). This defines the
maximum number of bits required to store the levels (levels are defined for all
-values in the column).</p><p>Two encodings for the levels are supported BIT_PACKED and RLE. Only RLE is now used as it supersedes BIT_PACKED.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cf811a6e949c01acbf67ad2370adc5c8>6 - Bloom Filter</h1><h3 id=problem-statement>Problem statement</h3><p>In their current format, column statistics and dictionaries can be used for predicate
+values in the column).</p><p>Two encodings for the levels are supported BIT_PACKED and RLE. Only RLE is now used as it supersedes BIT_PACKED.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cf811a6e949c01acbf67ad2370adc5c8>7 -</h1><h1 id=parquet-bloom-filter>Parquet Bloom Filter</h1><h3 id=problem-statement>Problem statement</h3><p>In their current format, column statistics and dictionaries can be used for predicate
pushdown. Statistics include minimum and maximum value, which can be used to filter out
values not in the range. Dictionaries are more specific, and readers can filter out values
that are between min and max but not in the dictionary. However, when there are too many
@@ -193,7 +1065,7 @@ significant 32 bits.</p><pre tabindex=0><code>void filter_insert(SBBF filter, un
block b = filter.getBlock(i);
return block_check(b, (unsigned int32)x)
}
-</code></pre><p>The use of blocks is from Putze et al.’s <a href=https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf>Cache-, Hash- and
+</code></pre><p>The use of blocks is from Putze et al.’s <a href=http://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf>Cache-, Hash- and
Space-Efficient Bloom
filters</a></p><p>To use an SBBF for values of arbitrary Parquet types, we apply a hash
function to that value - at the time of writing,
@@ -271,8 +1143,8 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src=/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
-<img src=/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
+<img src=/parquet-format/doc/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
+<img src=/parquet-format/doc/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
data should be encrypted with the column key, and the Bloom filter of other (not sensitive) columns
do not need to be encrypted.</p><p>Bloom filters have two serializable modules - the PageHeader thrift structure (with its internal
@@ -282,17 +1154,17 @@ serialized Bitset.</p><p>For Bloom filters in sensitive columns, each of the two
serialization, and then written to the file. The encryption will be performed using the AES GCM
cipher, with the same column key, but with different AAD module types - “BloomFilter Header” (8)
and “BloomFilter Bitset” (9). The length of the encrypted buffer is written before the buffer, as
-described in the Parquet encryption specification.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cfdfda4ee78f351464c0ddf31e278a88>7 - Data Pages</h1><p>For data pages, the 3 pieces of information are encoded back to back, after the page
+described in the Parquet encryption specification.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cfdfda4ee78f351464c0ddf31e278a88>8 - Data Pages</h1><p>For data pages, the 3 pieces of information are encoded back to back, after the page
header. No padding is allowed in the data page.
In order we have:</p><ol><li>repetition levels data</li><li>definition levels data</li><li>encoded values</li></ol><p>The value of <code>uncompressed_page_size</code> specified in the header is for all the 3 pieces combined.</p><p>The encoded values for the data page is always required. The definition and repetition levels
are optional, based on the schema definition. If the column is not nested (i.e.
the path to the column has length 1), we do not encode the repetition levels (it would
always have the value 1). For data that is required, the definition levels are
skipped (if encoded, it will always have the value of the max definition level).</p><p>For example, in the case where the column is non-nested and required, the data in the
-page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p></div><div class=td-content style=page-break-before:always><h1 id=pg-1745a7620ee7ae9962c3e6baaa083e89>7.1 - Compression</h1><h2 id=overview>Overview</h2><p>Parquet allows the data block inside dictionary pages and data pages to
+page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p></div><div class=td-content style=page-break-before:always><h1 id=pg-1745a7620ee7ae9962c3e6baaa083e89>8.1 -</h1><h1 id=parquet-compression-definitions>Parquet compression definitions</h1><p>This document contains the specification of all supported compression codecs.</p><h2 id=overview>Overview</h2><p>Parquet allows the dat [...]
be compressed for better space efficiency. The Parquet format supports
-several compression covering different areas in the compression ratio /
-processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
+several compression codecs covering different areas in the compression
+ratio / processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
by their respective authors or maintainers, which we reference hereafter.</p><p>For all compression codecs except the deprecated <code>LZ4</code> codec, the raw data
of a (data or dictionary) page is fed <em>as-is</em> to the underlying compression
library, without any additional framing or padding. The information required
@@ -322,8 +1194,8 @@ when implementing this format, the implementation provided by the
<a href=https://facebook.github.io/zstd/>Zstandard compression library</a>
is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md>LZ4 block format</a>.
If any ambiguity arises when implementing this format, the implementation
-provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>7.2 - Encodings</h1><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supported for types. It is
-intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding cannot be used. It
+provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>8.2 -</h1><h1 id=parquet-encoding-definitions>Parquet encoding definitions</h1><p>This file contains the specification of all supported encodings.</p><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be suppor [...]
+intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
@@ -350,8 +1222,7 @@ repeated-value := value that is repeated, using a fixed-width of round-up-to-nex
</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
-significant. An example of the encoding is presented below. For comparison, the same case is shown
-in the example of the deprecated bit-packing encoding in the next section.</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 10001000 11000110 11111010
@@ -425,7 +1296,9 @@ by keeping track of the number of values read.</p><p>Subtractions in steps 1) an
will the corresponding additions when decoding. Overflow should be allowed
and handled as wrapping around in 2’s complement notation so that the original
values are correctly restituted. This may require explicit care in some programming
-languages (for example by doing all arithmetic in the unsigned domain).</p><p>The following examples use 8 as the block size to keep the examples short,
+languages (for example by doing all arithmetic in the unsigned domain). Writers
+must not use more bits when bit packing the miniblock data than would be required
+to PLAIN encode the physical type (e.g. INT32 data must not use more than 32 bits).</p><p>The following examples use 8 as the block size to keep the examples short,
but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2, 3, 4, 5</p><p>After step 1), we compute the deltas as:</p><p>1, 1, 1, 1</p><p>The minimum delta is 1 and after step 2, the relative deltas become:</p><p>0, 0, 0, 0</p><p>The final encoded data is:</p><p>header:
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
@@ -437,11 +1310,10 @@ concatenated back to back. The expected savings is from the cost of encoding the
and possibly better compression in the data (it is no longer interleaved with the lengths).</p><p>The data stream looks like:</p><pre tabindex=0><code><Delta Encoded Lengths> <Byte Array Data>
</code></pre><p>For example, if the data was “Hello”, “World”, “Foobar”, “ABCDEF”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(5, 5, 6, 6) (the string lengths)</li><li>“HelloWorldFoobarABCDEF”</li></ul><h3 id=delta-strings-delta_byte_array--7>Delta Strings: (DELTA_BYTE_ARRAY = 7)</h3><p>Supported Types: BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY</p><p>This is also known as incremental [...]
sequence of strings, store the prefix length of the previous entry plus the suffix.</p><p>For a longer description, see <a href=https://en.wikipedia.org/wiki/Incremental_encoding>https://en.wikipedia.org/wiki/Incremental_encoding</a>.</p><p>This is stored as a sequence of delta-encoded prefix lengths (DELTA_BINARY_PACKED), followed by
-the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
+the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
compression ratio and speed when a compression algorithm is used afterwards.</p><p>This encoding creates K byte-streams of length N where K is the size in bytes of the data
-type and N is the number of elements in the data sequence. Specifically, K is 4 for FLOAT
-type and 8 for DOUBLE type.
-The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
+type and N is the number of elements in the data sequence. For example, K is 4 for FLOAT
+type and 8 for DOUBLE type.</p><p>The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
0-th stream, the 1-st byte goes to the 1-st stream and so on.
The streams are concatenated in the following order: 0-th stream, 1-st stream, etc.
The total length of encoded streams is K * N bytes. Because it does not have any metadata
@@ -450,7 +1322,7 @@ is allowed inside the data page.</p><p>Example:
Original data is three 32-bit floats and for simplicity we look at their raw representation.</p><pre tabindex=0><code> Element 0 Element 1 Element 2
Bytes AA BB CC DD 00 11 22 33 A3 B4 C5 D6
</code></pre><p>After applying the transformation, the data has the following representation:</p><pre tabindex=0><code>Bytes AA 00 A3 BB 11 B4 CC 22 C5 DD 33 D6
-</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-70f861f07d9aae8f8d133f3351caad43>7.3 - Parquet Modular Encryption</h1><p>Parquet files containing sensitive information can be protected by the modular encryption
+</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-70f861f07d9aae8f8d133f3351caad43>8.3 -</h1><h1 id=parquet-modular-encryption>Parquet Modular Encryption</h1><p>Parquet files containing sensitive information can be protected by the modular encryption
mechanism that encrypts and authenticates the file data and metadata - while allowing
for a regular Parquet functionality (columnar projection, predicate pushdown, encoding
and compression).</p><h2 id=1-problem-statement>1 Problem Statement</h2><p>Existing data protection solutions (such as flat encryption of files, in-storage encryption,
@@ -505,7 +1377,20 @@ The unique input to CTR encryption is called IV (“initialization vector&rd
parts: a nonce and an initial counter field.</p><p>Parquet encryption uses the RBG-based (random bit generator) nonce construction as defined in
the section 8.2.2 of the NIST SP 800-38D document. For each encrypted module, Parquet generates a
unique nonce with a length of 12 bytes (96 bits). Notice: the NIST
-specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
+specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h4 id=414-invocation-limit>4.1.4 Invocation limit</h4><p>According to the section 8.3 of the NIST SP 800-38D document, <em>“The total number of invocations
+of the authenticated encryption function shall not exceed 2^32, including all IV lengths and
+all instances of the authenticated encryption function with the given key”</em>. This restriction is
+related to the “uniqueness requirement of IVs and keys” (section 8 in the NIST spec) - <em>“if even
+one IV is ever repeated, then the implementation may be vulnerable”</em>. <em>“Compliance with this
+requirement is crucial to the security of GCM”</em>.</p><p>The bulk of modules in a Parquet file are page headers and data pages. Therefore, one encryption
+key shall not not be used for more than 2^31 (~2 billion) pages. In Parquet files encrypted with
+multiple keys (footer and column keys), the constraint on the number of invocations is applied
+to each key separately.</p><p>When running in the context of a larger system, any particular Parquet writer implementation likely
+does not have sufficient context to enforce key invocation limits system-wide. Therefore,
+the higher level system itself must arrange to supply keys appropriately to the various writer instances.</p><p>Parquet writer implementations should have a local invocation counter for each encryption key. If the
+counter exceeds 2^32, the implementation should return an error and produce no more cipherblocks.
+While this does not enforce a system-wide limit, it helps in distributed systems that provide different
+keys to different nodes (or generate unique keys in each node).</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
must be implemented by a cryptographic provider according to the NIST SP 800-38D specification.</p><p>In Parquet, an input to the GCM cipher is an encryption key, a 12-byte nonce, a plaintext and an
AAD. The output is a ciphertext with the length equal to that of plaintext, and a 16-byte authentication
tag used to verify the ciphertext and AAD integrity.</p><h4 id=422-aes_gcm_ctr_v1>4.2.2 AES_GCM_CTR_V1</h4><p>In this Parquet algorithm, all modules except pages are encrypted with the GCM cipher, as described
@@ -552,7 +1437,7 @@ data set (table). This string is optionally passed by a writer upon file creatio
the AAD prefix is stored in an <code>aad_prefix</code> field in the file, and is made available to the readers.
This field is not encrypted. If a user is concerned about keeping the file identity inside the file,
the writer code can explicitly request Parquet not to store the AAD prefix. Then the aad_prefix field
-will be empty; AAD prefixes must be fully managed by the caller code and supplied explicitly to Parquet
+will be empty; AAD prefixes must be fully managed by the caller code and supplied explictly to Parquet
readers for each file.</p><p>The protection against swapping full files is optional. It is not enabled by default because
it requires the writers to generate and pass an AAD prefix.</p><p>A reader of a file created with an AAD prefix, should be able to verify the prefix (file identity)
by comparing it with e.g. the target table name, using a convention accepted in the organization.
@@ -608,7 +1493,10 @@ decrypt it.</p><p>The row group ordinal, required for AAD suffix calculation, is
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/** Row group ordinal in the file **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>i16</span> <span style=color:#000>ordinal</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
+</span></span></code></pre></div><p>The integrity of this field is protected by authenticated encryption of the footer (FileMetaData). Therefore,
+the reader implementations can use either a local row group counter (ordinal) or the <code>RowGroup.ordinal</code>
+field as an input to AAD suffix calculation. The latter option can be helpful when different reader
+threads process different row groups in the same parquet file.</p><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
is a union - the actual structure is chosen depending on whether the column is encrypted with the
footer encryption key, or with a column-specific key. For the latter, a key metadata can be specified.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span> <span style=color:#000>EncryptionWithFooterKey</span> <span style=color:#000;font-weight:700>{</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
@@ -682,7 +1570,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img src=/parquet-format/doc/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -705,7 +1593,7 @@ structure with the
AES GCM algorithm - using a footer signing key, and an AAD constructed according to the instructions
of the section 4.4. Only the nonce and GCM tag are stored in the file – as a 28-byte
fixed-length array, written right after the footer itself. The ciphertext is not stored,
-because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</ [...]
+because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>stru [...]
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>...</span>
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/**
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * Encryption algorithm. This field is set only in encrypted files
@@ -726,17 +1614,17 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img src=/parquet-format/doc/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
to the default page size (1 MB). This is a rough estimation, and can change with the encryption
algorithm (no 16-byte tag in AES_GCM_CTR_V1) and with page configuration or data encoding/compression.</p><p>The throughput overhead of Parquet modular encryption depends on whether AES enciphering is
done in software or hardware. In both cases, performing encryption on full pages (~1MB buffers)
-instead of on much smaller individual data values causes AES to work at its maximal speed.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e4790b30bda91ce23679bbaa9c652cd0>7.4 - Checksumming</h1><p>Pages of all kinds can be individually checksummed. This allows disabling of checksums
+instead of on much smaller individual data values causes AES to work at its maximal speed.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e4790b30bda91ce23679bbaa9c652cd0>8.4 - Checksumming</h1><p>Pages of all kinds can be individually checksummed. This allows disabling of checksums
at the HDFS file level, to better support single row lookups. Checksums are calculated
using the standard CRC32 algorithm - as used in e.g. GZip - on the serialized binary
-representation of a page (not including the page header itself).</p></div><div class=td-content style=page-break-before:always><h1 id=pg-d84ba941c88c28699d637073c804ef79>7.5 - Column Chunks</h1><p>Column chunks are composed of pages written back to back. The pages share a common
+representation of a page (not including the page header itself).</p></div><div class=td-content style=page-break-before:always><h1 id=pg-d84ba941c88c28699d637073c804ef79>8.5 - Column Chunks</h1><p>Column chunks are composed of pages written back to back. The pages share a common
header and readers can skip over pages they are not interested in. The data for the
page follows the header and can be compressed and/or encoded. The compression and
encoding is specified in the page metadata.</p><p>A column chunk might be partly or completely dictionary encoded. It means that
@@ -745,7 +1633,7 @@ actual values are stored in the dictionary page. See details in Encodings.md.
The dictionary page must be placed at the first position of the column chunk. At
most one dictionary page can be placed in a column chunk.</p><p>Additionally, files can contain an optional column index to allow readers to
skip pages more efficiently. See PageIndex.md for details and
-the reasoning behind adding these to the format.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-3fa634598323d0f5132aedfc3988d28f>7.6 - Error Recovery</h1><p>If the file metadata is corrupt, the file is lost. If the column metadata is corrupt,
+the reasoning behind adding these to the format.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-3fa634598323d0f5132aedfc3988d28f>8.6 - Error Recovery</h1><p>If the file metadata is corrupt, the file is lost. If the column metadata is corrupt,
that column chunk is lost (but column chunks for this column in other row groups are
okay). If a page header is corrupt, the remaining pages in that chunk are lost. If
the data within a page is corrupt, that page is lost. The file will be more
@@ -755,12 +1643,14 @@ data written will be unreadable. This can be fixed by writing the file metadata
every Nth row group.
Each file metadata would be cumulative and include all the row groups written so
far. Combining this with the strategy used for rc or avro files using sync markers,
-a reader could recover partially written files.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-2c0e785302c1130bf2c45e6faa8aa18f>8 - Nulls</h1><p>Nullity is encoded in the definition levels (which is run-length encoded). NULL values
+a reader could recover partially written files.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-2c0e785302c1130bf2c45e6faa8aa18f>9 - Nulls</h1><p>Nullity is encoded in the definition levels (which is run-length encoded). NULL values
are not encoded in the data. For example, in a non-nested schema, a column with 1000 NULLs
would be encoded with run-length encoding (0, 1000 times) for the definition levels and
-nothing else.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-989662b6cba2c5cccd73a12b56828916>9 - Page Index</h1><p>This document describes the format for column index pages in the Parquet
-footer. These pages contain statistics for DataPages and can be used to skip
-pages when scanning data in ordered and unordered columns.</p><h2 id=problem-statement>Problem Statement</h2><p>In previous versions of the format, Statistics are stored for ColumnChunks in
+nothing else.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-989662b6cba2c5cccd73a12b56828916>10 -</h1><h1 id=parquet-page-index-layout-to-support-page-skipping>Parquet page index: Layout to Support Page Skipping</h1><p>In Parquet, a <em>page index</em> is optional metadata for a
+ColumnChunk, containing statistics for DataPages that can be used
+to skip those pages when scanning in ordered and unordered columns.
+The page index is stored using the OffsetIndex and ColumnIndex structures,
+defined in <a href=src/main/thrift/parquet.thrift><code>parquet.thrift</code></a></p><h2 id=problem-statement>Problem Statement</h2><p>In previous versions of the format, Statistics are stored for ColumnChunks in
ColumnMetaData and for individual pages inside DataPageHeader structs. When
reading pages, a reader had to process the page header to determine
whether the page could be skipped based on the statistics. This means the reader
@@ -782,7 +1672,7 @@ column are skipped, the corresponding rows in the other columns have to be
skipped. Hence the OffsetIndexes for each column in a RowGroup are stored
together.</li></ul><p>The new index structures are stored separately from RowGroup, near the footer.<br>This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
-location and length are stored in ColumnChunk.</p><p><img src=/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
+location and length are stored in ColumnChunk.</p><p><img src=/parquet-format/doc/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
bound for the last page, because the row group Statistics can provide that.
We still include those for the sake of uniformity, and the overhead should be
negligible.</li><li>We store lower and upper bounds for the values of each page. These may be the
@@ -800,8 +1690,8 @@ reader can find matching pages by sequentially reading <code>min_values</code> a
indices, and page offsets to scan in each column. The reader can then
initialize a scanner for each column and fast forward them to the start row of
the scan.</p><p>The <code>min_values</code> and <code>max_values</code> are calculated based on the <code>column_orders</code>
-field in the <code>FileMetaData</code> struct of the footer.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e0ad5830788d45de8b55e0c2b119349a>10 - Implementation status</h1><p>This page summarizes the features supported by different Parquet
+field in the <code>FileMetaData</code> struct of the footer.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e0ad5830788d45de8b55e0c2b119349a>11 - Implementation status</h1><p>This page summarizes the features supported by different Parquet
implementations.</p><p><em>Note</em>: If you find out of date information, please help us improve the accuracy
of this page by opening an issue or submitting a pull request.</p><h3 id=legend>Legend</h3><p>The value in each box means:</p><ul><li>✅: supported</li><li>❌: not supported</li><li>(R/W): partial reader/writer only support</li><li>(blank): no data</li></ul><p>Implementations:</p><ul><li><a href=https://github.com/apache/arrow/tree/main/cpp/src/parquet>arrow</a> (C++)</li><li><a href=https://github.com/apache/parquet-java>parquet-java</a> (Java)</li><li><a href=https://github.com/apache/ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/file-format/types/index.html b/output/_print/docs/file-format/types/index.html
index 42a39f9..ebc7ed4 100644
--- a/output/_print/docs/file-format/types/index.html
+++ b/output/_print/docs/file-format/types/index.html
@@ -2,8 +2,8 @@
- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><meta property="og:url" content="/docs/file-format/types/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Types"><met [...]
- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><meta property="og:locale" content="en"><meta property="og:type" content="website"><meta itemprop=name content="Types"><meta itemprop=description content [...]
- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><meta itemprop=dateModified content="2025-05-20T11:17:08+03:00"><meta itemprop=wordCount content="119"><meta name=twitter:card content="summary"><meta na [...]
-- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ [...]
-<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/types/>Return to the regular view of this page</a>.</p></div><h1 class=title>Types</h1><ul><li>1: <a href=#pg-967cf8be84f5efbfd2e57475707b79b1>Logical Types</a></li></ul><div class=content><p>The types supported by the file format are intended to be as minimal as possible,
+- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/ [...]
+<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/file-format/types/>Return to the regular view of this page</a>.</p></div><h1 class=title>Types</h1><ul><li>1: <a href=#pg-95b2ce86b5e36722017e238af2bd0fd8></a></li><li>2: <a href=#pg-967cf8be84f5efbfd2e57475707b79b1></a></li><li>3: <a href=#pg-29a57736b5ceb62cc6d5a0ba8ba429b0></a></li><li>4: <a href=#pg-ad7f0d170ee709f2e58f9e27d7567421></a></li></ul><div class=content><p>The types supported by the file for [...]
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
32-bit ints with an efficient encoding. This reduces the complexity of implementing
@@ -15,12 +15,821 @@ readers and writers for the format. The types are:</p><pre tabindex=0><code> -
- DOUBLE: IEEE 64-bit floating point values
- BYTE_ARRAY: arbitrarily long byte arrays
- FIXED_LEN_BYTE_ARRAY: fixed length byte arrays
-</code></pre></div></div><div class=td-content style=page-break-before:always><h1 id=pg-967cf8be84f5efbfd2e57475707b79b1>1 - Logical Types</h1><p>Logical types are used to extend the types that parquet can be used to store,
+</code></pre></div></div><div class=td-content style=page-break-before:always><h1 id=pg-95b2ce86b5e36722017e238af2bd0fd8>1 -</h1><h1 id=geospatial-definitions>Geospatial Definitions</h1><p>This document contains the specification of geospatial types and statistics.</p><h1 id=background>Background</h1><p>The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and
+Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM,
+XYZM) are defined by <a href="https://portal.ogc.org/files/?artifact_id=25355">OpenGIS Implementation Specification for Geographic
+information - Simple feature access - Part 1: Common architecture</a>,
+from <a href=https://www.ogc.org/standard/sfa/>OGC(Open Geospatial Consortium)</a>.</p><p>The version of the OGC standard first used here is 1.2.1, but future versions
+may also be used if the WKB representation remains wire-compatible.</p><h2 id=coordinate-reference-system>Coordinate Reference System</h2><p>Coordinate Reference System (CRS) is a mapping of how coordinates refer to
+locations on Earth.</p><p>The default CRS <code>OGC:CRS84</code> means that the geospatial features must be stored
+in the order of longitude/latitude based on the WGS84 datum.</p><p>Custom CRS can be specified by a string value. It is recommended to use an
+identifier-based approach like <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>.</p><p>For geographic CRS, longitudes are bound by [-180, 180] and latitudes are bound
+by [-90, 90].</p><h2 id=edge-interpolation-algorithm>Edge Interpolation Algorithm</h2><p>An algorithm for interpolating edges, and is one of the following values:</p><ul><li><code>spherical</code>: edges are interpolated as geodesics on a sphere.</li><li><code>vincenty</code>: <a href=https://en.wikipedia.org/wiki/Vincenty%27s_formulae>https://en.wikipedia.org/wiki/Vincenty%27s_formulae</a></li><li><code>thomas</code>: Thomas, Paul D. Spheroidal geodesics, reference systems, & local geom [...]
+logical types to store statistics of a column chunk. It is an optional field in
+the <code>ColumnMetaData</code> and contains <a href=/docs/file-format/types/geospatial/#bounding-box>Bounding Box</a> and <a href=/docs/file-format/types/geospatial/#geospatial-types>Geospatial
+Types</a> that are described below in detail.</p><h2 id=bounding-box>Bounding Box</h2><p>A geospatial instance has at least two coordinate dimensions: X and Y for 2D
+coordinates of each point. Please note that X is longitude/easting and Y is
+latitude/northing. A geospatial instance can optionally have Z and/or M values
+associated with each point.</p><p>The Z values introduce the third dimension coordinate. Usually they are used to
+indicate the height, or elevation.</p><p>M values are an opportunity for a geospatial instance to track a value in a
+fourth dimension. These values can be used as a linear reference value (e.g.,
+highway milepost value), a timestamp, or some other value as defined by the CRS.</p><p>Bounding box is defined as the thrift struct below in the representation of
+min/max value pair of coordinates from each axis. Note that X and Y Values are
+always present. Z and M are omitted for 2D geospatial instances.</p><p>When calculating a bounding box, null or NaN values in a coordinate
+dimension are skipped. For example, <code>POINT (1 NaN)</code> contributes a value to X
+but no values to Y, Z, or M dimension of the bounding box. If a dimension has
+only null or NaN values, that dimension is omitted from the bounding box. If
+either the X or Y dimension is missing, then the bounding box itself is not
+produced.</p><p>For the X values only, xmin may be greater than xmax. In this case, an object
+in this bounding box may match if it contains an X such that <code>x >= xmin</code> OR
+<code>x <= xmax</code>. This wraparound occurs only when the corresponding bounding box
+crosses the antimeridian line. In geographic terminology, the concepts of <code>xmin</code>,
+<code>xmax</code>, <code>ymin</code>, and <code>ymax</code> are also known as <code>westernmost</code>, <code>easternmost</code>,
+<code>southernmost</code> and <code>northernmost</code>, respectively.</p><p>For <code>GEOGRAPHY</code> types, X and Y values are restricted to the canonical ranges of
+[-180, 180] for X and [-90, 90] for Y.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-thrift data-lang=thrift><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>BoundingBox</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000;font-weight:700>{</span><s [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>2</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>3</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>4</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>5</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>6</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>8</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#000;font-weight:700>}</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span></code></pre></div><h2 id=geospatial-types>Geospatial Types</h2><p>A list of geospatial types from all instances in the <code>GEOMETRY</code> or <code>GEOGRAPHY</code>
+column, or an empty list if they are not known.</p><p>This is borrowed from <a href="https://github.com/opengeospatial/geoparquet/blob/v1.1.0/format-specs/geoparquet.md?plain=1#L159">geometry_types of GeoParquet</a> except that
+values in the list are <a href=https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary>WKB (ISO-variant) integer codes</a>.
+Table below shows the most common geospatial types and their codes:</p><table><thead><tr><th style=text-align:left>Type</th><th style=text-align:left>XY</th><th style=text-align:left>XYZ</th><th style=text-align:left>XYM</th><th style=text-align:center>XYZM</th></tr></thead><tbody><tr><td style=text-align:left>Point</td><td style=text-align:left>0001</td><td style=text-align:left>1001</td><td style=text-align:left>2001</td><td style=text-align:center>3001</td></tr><tr><td style=text-alig [...]
+responsible for serializing and deserializing the CRS, respectively.</p><p>As a convention to maximize the interoperability, custom CRS values can be
+specified by a string of the format <code>type:identifier</code>, where <code>type</code> is one of
+the following values:</p><ul><li><code>srid</code>: <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>, <code>identifier</code> is the SRID itself.</li><li><code>projjson</code>: <a href=https://proj.org/en/stable/specifications/projjson.html>PROJJSON</a>, <code>identifier</code> is the name of a table property or a file property where the projjson string is stored.</li></ul><h1 id=coordinate-axis-order>Coordinate axis order</h1><p> [...]
+follows the de facto standard for axis order in WKB and is therefore always
+(x, y) where x is easting or longitude and y is northing or latitude. This
+ordering explicitly overrides the axis order as specified in the CRS.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-967cf8be84f5efbfd2e57475707b79b1>2 -</h1><h1 id=parquet-logical-type-definitions>Parquet Logical Type Definitions</h1><p>Logical types are used to extend the types that parquet can be used to store,
by specifying how the primitive types should be interpreted. This keeps the set
of primitive types to a minimum and reuses parquet’s efficient encodings. For
-example, strings are stored as byte arrays (binary) with a UTF8 annotation.
-These annotations define how to further decode and interpret the data.
-Annotations are stored as <code>LogicalType</code> fields in the file metadata and are
-documented in <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>LogicalTypes.md</a></p></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener href=mailto:dev@parquet [...]
+example, strings are stored with the primitive type <code>BYTE_ARRAY</code> with a <code>STRING</code>
+annotation.</p><p>This file contains the specification for all logical types.</p><h3 id=metadata>Metadata</h3><p>The parquet format’s <code>LogicalType</code> stores the type annotation. The annotation
+may require additional metadata fields, as well as rules for those fields.</p><p>There is an older representation of the logical type annotations called <code>ConvertedType</code>.
+To support backward compatibility with old files, readers should interpret <code>LogicalTypes</code>
+in the same way as <code>ConvertedType</code>, and writers should populate <code>ConvertedType</code> in the metadata
+according to well defined conversion rules.</p><h3 id=compatibility>Compatibility</h3><p>The Thrift definition of the metadata has two fields for logical types: <code>ConvertedType</code> and <code>LogicalType</code>.
+<code>ConvertedType</code> is an enum of all available annotations. Since Thrift enums can’t have additional type parameters,
+it is cumbersome to define additional type parameters, like decimal scale and precision
+(which are additional 32 bit integer fields on SchemaElement, and are relevant only for decimals) or time unit
+and UTC adjustment flag for Timestamp types. To overcome this problem, a new logical type representation was introduced into
+the metadata to replace <code>ConvertedType</code>: <code>LogicalType</code>. The new representation is a union of structs of logical types,
+this way allowing more flexible API, logical types can have type parameters.</p><p><code>ConvertedType</code> is deprecated. However, to maintain compatibility with old writers,
+Parquet readers should be able to read and interpret <code>ConvertedType</code> annotations
+in case <code>LogicalType</code> annotations are not present. Parquet writers must always write
+<code>LogicalType</code> annotations where applicable, but must also write the corresponding
+<code>ConvertedType</code> annotations (if any) to maintain compatibility with old readers.</p><p>Compatibility considerations are mentioned for each annotation in the corresponding section.</p><h2 id=string-types>String Types</h2><h3 id=string>STRING</h3><p><code>STRING</code> may only be used to annotate the <code>BYTE_ARRAY</code> primitive type and indicates
+that the byte array should be interpreted as a UTF-8 encoded character string.</p><p>The sort order used for <code>STRING</code> strings is unsigned byte-wise comparison.</p><p><em>Compatibility</em></p><p><code>STRING</code> corresponds to <code>UTF8</code> ConvertedType.</p><h3 id=enum>ENUM</h3><p><code>ENUM</code> annotates the <code>BYTE_ARRAY</code> primitive type and indicates that the value
+was converted from an enumerated type in another data model (e.g. Thrift, Avro, Protobuf).
+Applications using a data model lacking a native enum type should interpret <code>ENUM</code>
+annotated field as a UTF-8 encoded string.</p><p>The sort order used for <code>ENUM</code> values is unsigned byte-wise comparison.</p><h3 id=uuid>UUID</h3><p><code>UUID</code> annotates a 16-byte <code>FIXED_LEN_BYTE_ARRAY</code> primitive type. The value is
+encoded using big-endian, so that <code>00112233-4455-6677-8899-aabbccddeeff</code> is encoded
+as the bytes <code>00 11 22 33 44 55 66 77 88 99 aa bb cc dd ee ff</code>
+(This example is from <a href=https://en.wikipedia.org/wiki/Universally_unique_identifier>wikipedia’s UUID page</a>).</p><p>The sort order used for <code>UUID</code> values is unsigned byte-wise comparison.</p><h2 id=numeric-types>Numeric Types</h2><h3 id=signed-integers>Signed Integers</h3><p><code>INT</code> annotation can be used to specify the maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+For signed integers, the second parameter should be <code>true</code>,
+for example, a signed integer with bit width of 8 is defined as <code>INT(8, true)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, true)</code>, <code>INT(16, true)</code>, and <code>INT(32, true)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, true)</code> must annotate an <code>int64</code> primitive type. <code>INT(32, true)</code> and <code>INT(64, true)</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p>The sort order used for signed integer types is signed.</p><h3 id=unsigned-integers>Unsigned Integers</h3><p><code>INT</code> annotation can be used to specify unsigned integer types,
+along with a maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+In case of unsigned integers, the second parameter should be <code>false</code>,
+for example, an unsigned integer with bit width of 8 is defined as <code>INT(8, false)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, false)</code>, <code>INT(16, false)</code>, and <code>INT(32, false)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, false)</code> must annotate an <code>int64</code> primitive type.</p><p>The sort order used for unsigned integer types is unsigned.</p><h3 id=deprecated-integer-convertedtype>Deprecated integer ConvertedType</h3><p><code>INT_8</code>, <code>INT_16</code>, <code>INT_32</code>, and <code>INT_64</code> annotations can be also used to specify
+signed integers with 8, 16, 32, or 64 bit width.</p><p><code>INT_8</code>, <code>INT_16</code>, and <code>INT_32</code> must annotate an <code>int32</code> primitive type and
+<code>INT_64</code> must annotate an <code>int64</code> primitive type. <code>INT_32</code> and <code>INT_64</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p><code>UINT_8</code>, <code>UINT_16</code>, <code>UINT_32</code>, and <code>UINT_64</code> annotations can be also used to specify
+unsigned integers with 8, 16, 32, or 64 bit width.</p><p><code>UINT_8</code>, <code>UINT_16</code>, and <code>UINT_32</code> must annotate an <code>int32</code> primitive type and
+<code>UINT_64</code> must annotate an <code>int64</code> primitive type.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>INT_8</td><td>IntType (bitWidth = 8, isSigned = true)</td></tr><tr><td>INT_16</td><td>IntType (bitWidth = 16, isSigned = true)</td></tr><tr><td>INT_32</td><td>IntType (bitWidth = 32, isSigned = true)</td></tr><tr><td>INT_64</td><td>IntType (bitWidth = 64, isSigned = true)</td></tr><tr><td [...]
+the form <code>unscaledValue * 10^(-scale)</code>.</p><p>The primitive type stores an unscaled integer value. For <code>BYTE_ARRAY</code> and
+<code>FIXED_LEN_BYTE_ARRAY</code>, the unscaled number must be encoded as two’s complement using
+big-endian byte order (the most significant byte is the zeroth element). The
+scale stores the number of digits of that value that are to the right of the
+decimal point, and the precision stores the maximum number of digits supported
+in the unscaled value.</p><p>If not specified, the scale is 0. Scale must be zero or a positive integer less
+than or equal to the precision. Precision is required and must be a non-zero positive
+integer. A precision too large for the underlying type (see below) is an error.</p><p><code>DECIMAL</code> can be used to annotate the following types:</p><ul><li><code>int32</code>: for 1 <= precision <= 9</li><li><code>int64</code>: for 1 <= precision <= 18; precision < 10 will produce a
+warning</li><li><code>fixed_len_byte_array</code>: precision is limited by the array size. Length <code>n</code>
+can store <= <code>floor(log_10(2^(8*n - 1) - 1))</code> base-10 digits</li><li><code>byte_array</code>: <code>precision</code> is not limited, but is required. The minimum number of
+bytes to store the unscaled value should be used.</li></ul><p>The sort order used for <code>DECIMAL</code> values is signed comparison of the represented
+value.</p><p>If the column uses <code>int32</code> or <code>int64</code> physical types, then signed comparison of
+the integer values produces the correct ordering. If the physical type is
+fixed, then the correct ordering can be produced by flipping the
+most-significant bit in the first byte and then using unsigned byte-wise
+comparison.</p><p><em>Compatibility</em></p><p>To support compatibility with older readers, implementations of parquet-format should
+write <code>DecimalType</code> precision and scale into the corresponding SchemaElement field in metadata.</p><h3 id=float16>FLOAT16</h3><p>The <code>FLOAT16</code> annotation represents half-precision floating-point numbers in the 2-byte IEEE little-endian format.</p><p>Used in contexts where precision is traded off for smaller footprint and potentially better performance.</p><p>The primitive type is a 2-byte <code>FIXED_LEN_BYTE_ARRAY</code>.</p><p>The sort order for <code>FLOAT16</cod [...]
+annotate an <code>int32</code> that stores the number of days from the Unix epoch, 1
+January 1970.</p><p>The sort order used for <code>DATE</code> is signed.</p><h3 id=time>TIME</h3><p><code>TIME</code> is used for a logical time type without a date with millisecond or microsecond precision.
+The type has two type parameters: UTC adjustment (<code>true</code> or <code>false</code>)
+and unit (<code>MILLIS</code> or <code>MICROS</code>, <code>NANOS</code>).</p><p><code>TIME</code> with unit <code>MILLIS</code> is used for millisecond precision.
+It must annotate an <code>int32</code> that stores the number of
+milliseconds after midnight.</p><p><code>TIME</code> with unit <code>MICROS</code> is used for microsecond precision.
+It must annotate an <code>int64</code> that stores the number of
+microseconds after midnight.</p><p><code>TIME</code> with unit <code>NANOS</code> is used for nanosecond precision.
+It must annotate an <code>int64</code> that stores the number of
+nanoseconds after midnight.</p><p>The sort order used for <code>TIME</code> is signed.</p><h4 id=deprecated-time-convertedtype>Deprecated time ConvertedType</h4><p><code>TIME_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int32</code>.</p><p><code>TIME_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MICROS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local time semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local time with legacy <code>TIME_MICROS</code> and <code>TIME_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local time with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIME_MILLIS</td><td>TimeType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIME_MICROS</td><td>TimeType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimeType</td><td rowspan=3>isAdj [...]
+<code>int64</code> number that can be decoded into year, month, day, hour, minute, second
+and subsecond fields using calculations detailed below. Please note that a value
+defined this way does not necessarily correspond to a single instant on the
+time-line and such interpretations are allowed on purpose.</p><p>The <code>TIMESTAMP</code> type has two type parameters:</p><ul><li><code>isAdjustedToUTC</code> must be either <code>true</code> or <code>false</code>.</li><li><code>unit</code> must be one of <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>. This list is subject
+to potential expansion in the future. Upon reading, unknown <code>unit</code>-s must
+be handled as unsupported features (rather than as errors in the data files).</li></ul><h4 id=instant-semantics-timestamps-normalized-to-utc>Instant semantics (timestamps normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=true</code> is defined as the number of
+milliseconds, microseconds or nanoseconds (depending on the <code>unit</code>
+parameter being <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>, respectively) elapsed since the
+Unix epoch, 1970-01-01 00:00:00 UTC. Each such value unambiguously identifies a
+single instant on the time-line.</p><p>For example, in a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>, the
+number 172800000 corresponds to 1970-01-03 00:00:00 UTC, because it is equal to
+2 * 24 * 60 * 60 * 1000, therefore it is exactly two days from the reference
+point, the Unix epoch. In Java, this calculation can be achieved by calling
+<code>Instant.ofEpochMilli(172800000)</code>.</p><p>As a slightly more complicated example, if one wants to store 1970-01-03
+00:00:00 (UTC+01:00) as a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>,
+first the time zone offset has to be dealt with. By normalizing the timestamp to
+UTC, we calculate what time in UTC corresponds to the same instant: 1970-01-02
+23:00:00 UTC. This is 1 day and 23 hours after the epoch, therefore it can be
+encoded as the number (24 + 23) * 60 * 60 * 1000 = 169200000.</p><p>Please note that time zone information gets lost in this process. Upon reading a
+value back, we can only reconstruct the instant, but not the original
+representation. In practice, such timestamps are typically displayed to users in
+their local time zones, therefore they may be displayed differently depending on
+the execution environment.</p><h4 id=local-semantics-timestamps-not-normalized-to-utc>Local semantics (timestamps not normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=false</code> represents year, month, day, hour,
+minute, second and subsecond fields in a local timezone, <em>regardless of what
+specific time zone is considered local</em>. This means that such timestamps should
+always be displayed the same way, regardless of the local time zone in effect.
+On the other hand, without additional information such as an offset or
+time-zone, these values do not identify instants on the time-line unambiguously,
+because the corresponding instants would depend on the local time zone.</p><p>Using a single number to represent a local timestamp is a lot less intuitive
+than for instants. One must use a local timestamp as the reference point and
+calculate the elapsed time between the actual timestamp and the reference point.
+The problem is that the result may depend on the local time zone, for example
+because there may have been a daylight saving time change between the two
+timestamps.</p><p>The solution to this problem is to use a simplification that makes the result
+easy to calculate and independent of the timezone. Treating every day as
+consisting of exactly 86400 seconds and ignoring DST changes altogether allows
+us to unambiguously represent a local timestamp as a difference from a reference
+local timestamp. We define the reference local timestamp to be 1970-01-01
+00:00:00 (note the lack of UTC at the end, as this is not an instant). This way
+the encoding of local timestamp values becomes very similar to the encoding of
+instant values. For example, in a <code>TIMESTAMP(isAdjustedToUTC=false, unit=MILLIS)</code>, the number 172800000 corresponds to 1970-01-03 00:00:00
+(note the lack of UTC at the end), because it is exactly two days from the
+reference point (172800000 = 2 * 24 * 60 * 60 * 1000).</p><p>Another way to get to the same definition is to treat the local timestamp value
+<em>as if</em> it were in UTC and store it as an instant. For example, if we treat the
+local timestamp 1970-01-03 00:00:00 <em>as if</em> it were the instant 1970-01-03
+00:00:00 UTC, we can store it as 172800000. When reading 172800000 back, we can
+reconstruct the instant 1970-01-03 00:00:00 UTC and convert it to a local
+timestamp <em>as if</em> we were in the UTC time zone, resulting in 1970-01-03
+00:00:00. In Java, this can be achieved by calling
+<code>LocalDateTime.ofEpochSecond(172800, 0, ZoneOffset.UTC)</code>.</p><p>Please note that while from a practical point of view this second definition is
+equivalent to the first one, from a theoretical point of view only the first
+definition can be considered correct, the second one just “incidentally” leads
+to the same results. Nevertheless, this second definition is worth mentioning as
+well, because it is relatively widespread and it can lead to confusion,
+especially due to its usage of UTC in the calculations. One can stumble upon
+code, comments and specifications ambiguously stating that a timestamp “is
+stored in UTC”. In some contexts, it means that it is <em>normalized</em> to UTC and
+acts as an instant. In some other contexts though, it means the exact opposite,
+namely that the timestamp is stored <em>as if</em> it were in UTC and acts as a
+local timestamp in reality.</p><h4 id=common-considerations>Common considerations</h4><p>Every possible <code>int64</code> number represents a valid timestamp, but depending on the
+precision, the corresponding year may be outside of the practical everyday
+limits and implementations may choose to only support a limited range.</p><p>On the other hand, not every combination of year, month, day, hour, minute,
+second and subsecond values can be encoded into an <code>int64</code>. Most notably:</p><ul><li>An arbitrary combination of timestamp fields can not be encoded as a single
+number if the values for some of the fields are outside of their normal range
+(where the “normal range” corresponds to everyday usage). For example, neither
+of the following can be represented in a timestamp:<ul><li>hour = -1</li><li>hour = 25</li><li>minute = 61</li><li>month = 13</li><li>day = 29, month = 2, year = any non-leap year</li></ul></li><li>Due to the range of the <code>int64</code> type, timestamps using the <code>NANOS</code> unit
+can only represent values between 1677-09-21 00:12:43 and 2262-04-11 23:47:16.
+Values outside of this range can not be represented with the <code>NANOS</code>
+unit. (Other precisions have similar limits but those are outside of the
+domain for practical everyday usage.)</li></ul><p>The sort order used for <code>TIMESTAMP</code> is signed.</p><h4 id=deprecated-timestamp-convertedtype>Deprecated timestamp ConvertedType</h4><p><code>TIMESTAMP_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p><code>TIMESTAMP_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MICROS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local timestamp semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local timestamps with legacy <code>TIMESTAMP_MICROS</code> and <code>TIMESTAMP_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local timestamps with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIMESTAMP_MILLIS</td><td>TimestampType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIMESTAMP_MICROS</td><td>TimestampType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimestampTyp [...]
+<code>fixed_len_byte_array</code> of length 12. This array stores three little-endian
+unsigned integers that represent durations at different granularities of time.
+The first stores a number in months, the second stores a number in days, and
+the third stores a number in milliseconds. This representation is independent
+of any particular timezone or date.</p><p>Each component in this representation is independent of the others. For
+example, there is no requirement that a large number of days should be
+expressed as a mix of months and days because there is not a constant
+conversion from days to months.</p><p>The sort order used for <code>INTERVAL</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=embedded-types>Embedded Types</h2><p>Embedded types do not have type-specific orderings.</p><h3 id=json>JSON</h3><p><code>JSON</code> is used for an embedded JSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as a UTF-8 encoded character
+string of valid JSON as defined by the <a href=http://json.org/>JSON specification</a></p><p>The sort order used for <code>JSON</code> is unsigned byte-wise comparison.</p><h3 id=bson>BSON</h3><p><code>BSON</code> is used for an embedded BSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as an encoded BSON document as
+defined by the <a href=http://bsonspec.org/spec.html>BSON specification</a>.</p><p>The sort order used for <code>BSON</code> is unsigned byte-wise comparison.</p><h3 id=variant>VARIANT</h3><p><code>VARIANT</code> is used for a Variant value. It must annotate a group. The group must
+contain a field named <code>metadata</code> and a field named <code>value</code>. Both fields must have
+type <code>binary</code>, which is also called <code>BYTE_ARRAY</code> in the Parquet thrift definition.
+The <code>VARIANT</code> annotated group can be used to store either an unshredded Variant
+value, or a shredded Variant value.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</code> must be of type <code>binary</code> (called <code>BYTE_ARRAY</code>
+in the Parquet thrift definition).</li><li>The <code>metadata</code> field is required and must be a valid Variant metadata component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>When present, the <code>value</code> field must be a valid Variant value component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>The <code>value</code> field is required for unshredded Variant values.</li><li>The <code>value</code> field is optional and may be null only when parts of the Variant
+value are shredded according to the <a href=/docs/file-format/types/variantshredding/>Variant shredding specification</a>.</li></ul><p>This is the expected representation of an unshredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_unshredded (VARIANT) {
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_shredded (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><h3 id=geometry>GEOMETRY</h3><p><code>GEOMETRY</code> is used for geospatial features in the Well-Known Binary (WKB) format
+with linear/planar edges interpolation. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has only one type parameter:</p><ul><li><code>crs</code>: An optional string value for CRS. If unset, the CRS defaults to
+<code>"OGC:CRS84"</code>, which means that the geometries must be stored in longitude,
+latitude based on the WGS84 datum.</li></ul><p>The sort order used for <code>GEOMETRY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h3 id=geography>GEOGRAPHY</h3><p><code>GEOGRAPHY</code> is used for geospatial features in the WKB format with an explicit
+(non-linear/non-planar) edges interpolation algorithm. It must annotate a
+<code>BYTE_ARRAY</code> primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has two type parameters:</p><ul><li><code>crs</code>: An optional string value for CRS. It must be a geographic CRS, where
+longitudes are bound by [-180, 180] and latitudes are bound by [-90, 90].
+If unset, the CRS defaults to <code>"OGC:CRS84"</code>.</li><li><code>algorithm</code>: An optional enum value to describes the edge interpolation
+algorithm. Supported values are: <code>SPHERICAL</code>, <code>VINCENTY</code>, <code>THOMAS</code>, <code>ANDOYER</code>,
+<code>KARNEY</code>. If unset, the algorithm defaults to <code>SPHERICAL</code>.</li></ul><p>The sort order used for <code>GEOGRAPHY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=nested-types>Nested Types</h2><p>This section specifies how <code>LIST</code> and <code>MAP</code> can be used to encode nested types
+by adding group levels around repeated fields that are not present in the data.</p><p>This does not affect repeated fields that are not annotated: A repeated field
+that is neither contained by a <code>LIST</code>- or <code>MAP</code>-annotated group nor annotated
+by <code>LIST</code> or <code>MAP</code> should be interpreted as a required list of required
+elements where the element type is the type of the field.</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// List<Integer> (non-null list, non-null elements)
+repeated int32 num;
+
+// List<Tuple<Integer, String>> (non-null list, non-null elements)
+repeated group my_list {
+ required int32 num;
+ optional binary str (STRING);
+}
+</code></pre><p>For all fields in the schema, implementations should use either <code>LIST</code> and
+<code>MAP</code> annotations <em>or</em> unannotated repeated fields, but not both. When using
+the annotations, no unannotated repeated types are allowed.</p><h3 id=lists>Lists</h3><p><code>LIST</code> is used to annotate types that should be interpreted as lists.</p><p><code>LIST</code> must always annotate a 3-level structure:</p><pre tabindex=0><code><list-repetition> group <name> (LIST) {
+ repeated group list {
+ <element-repetition> <element-type> element;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>LIST</code> that contains a
+single field named <code>list</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the list is nullable.</li><li>The middle level, named <code>list</code>, must be a repeated group with a single
+field named <code>element</code>.</li><li>The <code>element</code> field encodes the list’s element type and repetition. Element
+repetition must be <code>required</code> or <code>optional</code>.</li></ul><p>The following examples demonstrate two of the possible lists of string values.</p><pre tabindex=0><code>// List<String> (list non-null, elements nullable)
+required group my_list (LIST) {
+ repeated group list {
+ optional binary element (STRING);
+ }
+}
+
+// List<String> (list nullable, elements non-null)
+optional group my_list (LIST) {
+ repeated group list {
+ required binary element (STRING);
+ }
+}
+</code></pre><p>Element types can be nested structures. For example, a list of lists:</p><pre tabindex=0><code>// List<List<Integer>>
+optional group array_of_arrays (LIST) {
+ repeated group list {
+ required group element (LIST) {
+ repeated group list {
+ required int32 element;
+ }
+ }
+ }
+}
+</code></pre><h4 id=backward-compatibility-rules>Backward-compatibility rules</h4><p>New writer implementations should always produce the 3-level LIST structure shown
+above. However, historically data files have been produced that use different
+structures to represent list-like data, and readers may include compatibility
+measures to interpret them as intended.</p><p>It is required that the repeated group of elements is named <code>list</code> and that
+its element field is named <code>element</code>. However, these names may not be used in
+existing data and should not be enforced as errors when reading. For example,
+the following field schema should produce a nullable list of non-null strings,
+even though the repeated group is named <code>element</code>.</p><pre tabindex=0><code>optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ };
+}
+</code></pre><p>Some existing data does not include the inner element layer, resulting in a
+<code>LIST</code> that annotates a 2-level structure. Unlike the 3-level structure, the
+repetition of a 2-level structure can be <code>optional</code>, <code>required</code>, or <code>repeated</code>.
+When it is <code>repeated</code>, the <code>LIST</code>-annotated 2-level structure can only serve as
+an element within another <code>LIST</code>-annotated 2-level structure.</p><p>For backward-compatibility, the type of elements in <code>LIST</code>-annotated structures
+should always be determined by the following rules:</p><ol><li>If the repeated field is not a group, then its type is the element type and
+elements are required.</li><li>If the repeated field is a group with multiple fields, then its type is the
+element type and elements are required.</li><li>If the repeated field is a group with one field with <code>repeated</code> repetition,
+then its type is the element type and elements are required.</li><li>If the repeated field is a group with one field and is named either <code>array</code>
+or uses the <code>LIST</code>-annotated group’s name with <code>_tuple</code> appended then the
+repeated type is the element type and elements are required.</li><li>Otherwise, the repeated field’s type is the element type with the repeated
+field’s repetition.</li></ol><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// Rule 1: List<Integer> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated int32 element;
+}
+
+// Rule 2: List<Tuple<String, Integer>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ required int32 num;
+ };
+}
+
+// Rule 3: List<List<Integer>> (nullable outer list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array (LIST) {
+ repeated int32 array;
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array {
+ required binary str (STRING);
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group my_list_tuple {
+ required binary str (STRING);
+ };
+}
+
+// Rule 5: List<String> (nullable list, nullable elements)
+optional group my_list (LIST) {
+ repeated group element {
+ optional binary str (STRING);
+ };
+}
+</code></pre><h3 id=maps>Maps</h3><p><code>MAP</code> is used to annotate types that should be interpreted as a map from keys
+to values. <code>MAP</code> must annotate a 3-level structure:</p><pre tabindex=0><code><map-repetition> group <name> (MAP) {
+ repeated group key_value {
+ required <key-type> key;
+ <value-repetition> <value-type> value;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>MAP</code> that contains a
+single field named <code>key_value</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the map is nullable.</li><li>The middle level, named <code>key_value</code>, must be a repeated group with a <code>key</code>
+field for map keys and, optionally, a <code>value</code> field for map values. It must
+not contain any other values.</li><li>The <code>key</code> field encodes the map’s key type. This field must have
+repetition <code>required</code> and must always be present. It must always be the first
+field of the repeated <code>key_value</code> group.</li><li>The <code>value</code> field encodes the map’s value type and repetition. This field can
+be <code>required</code>, <code>optional</code>, or omitted. It must always be the second field of
+the repeated <code>key_value</code> group if present. In case of not present, it can be
+represented as a map with all null values or as a set of keys.</li></ul><p>The following example demonstrates the type for a non-null map from strings to
+nullable integers:</p><pre tabindex=0><code>// Map<String, Integer>
+required group my_map (MAP) {
+ repeated group key_value {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><p>If there are multiple key-value pairs for the same key, then the final value
+for that key must be the last value. Other values may be ignored or may be
+added with replacement to the map container in the order that they are encoded.
+The <code>MAP</code> annotation should not be used to encode multi-maps using duplicate
+keys.</p><h4 id=backward-compatibility-rules-1>Backward-compatibility rules</h4><p>It is required that the repeated group of key-value pairs is named <code>key_value</code>
+and that its fields are named <code>key</code> and <code>value</code>. However, these names may not
+be used in existing data and should not be enforced as errors when reading.
+(<code>key</code> and <code>value</code> can be identified by their position in case of misnaming.)</p><p>Some existing data incorrectly used <code>MAP_KEY_VALUE</code> in place of <code>MAP</code>. For
+backward-compatibility, a group annotated with <code>MAP_KEY_VALUE</code> that is not
+contained by a <code>MAP</code>-annotated group should be handled as a <code>MAP</code>-annotated
+group.</p><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>// Map<String, Integer> (nullable map, non-null values)
+optional group my_map (MAP) {
+ repeated group map {
+ required binary str (STRING);
+ required int32 num;
+ }
+}
+
+// Map<String, Integer> (nullable map, nullable values)
+optional group my_map (MAP_KEY_VALUE) {
+ repeated group map {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><h2 id=unknown-always-null>UNKNOWN (always null)</h2><p>Sometimes, when discovering the schema of existing data, values are always null
+and there’s no type information.
+The <code>UNKNOWN</code> type can be used to annotate a column that is always null.
+(Similar to Null type in Avro and Arrow)</p></div><div class=td-content style=page-break-before:always><h1 id=pg-29a57736b5ceb62cc6d5a0ba8ba429b0>3 -</h1><h1 id=variant-shredding>Variant Shredding</h1><p>The Variant type is designed to store and process semi-structured data efficiently, even with heterogeneous values.
+Query engines encode each Variant value in a self-describing format, and store it as a group containing <code>value</code> and <code>metadata</code> binary fields in Parquet.
+Since data is often partially homogeneous, it can be beneficial to extract certain fields into separate Parquet columns to further improve performance.
+This process is called <strong>shredding</strong>.</p><p>Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections.</p><p>For example, the query <code>SELECT variant_get(event, '$.event_ts', 'timestamp') FROM tbl</code> only needs to load field <code>event_ts</code>, and if that column is shredded, it can be read by columnar projection without reading or deserializing the rest of the < [...]
+Similarly, for the query <code>SELECT * FROM tbl WHERE variant_get(event, '$.event_type', 'string') = 'signup'</code>, the <code>event_type</code> shredded column metadata can be used for skipping and to lazily load the rest of the Variant.</p><h2 id=variant-metadata>Variant Metadata</h2><p>Variant metadata is stored in the top-level Variant group in a binary <code>metadata</code> column regardless of whether the Variant value is shredded.</p><p>All <code>value</code> columns within the [...]
+All field names of a Variant, whether shredded or not, must be present in the metadata.</p><h2 id=value-shredding>Value Shredding</h2><p>Variant values are stored in Parquet fields named <code>value</code>.
+Each <code>value</code> field may have an associated shredded field named <code>typed_value</code> that stores the value when it matches a specific type.
+When <code>typed_value</code> is present, readers <strong>must</strong> reconstruct shredded values according to this specification.</p><p>For example, a Variant field, <code>measurement</code> may be shredded as long values by adding <code>typed_value</code> with type <code>int64</code>:</p><pre tabindex=0><code>required group measurement (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The Parquet columns used to store variant metadata and values must be accessed by name, not by position.</p><p>The series of measurements <code>34, null, "n/a", 100</code> would be stored as:</p><table><thead><tr><th>Value</th><th><code>metadata</code></th><th><code>value</code></th><th><code>typed_value</code></th></tr></thead><tbody><tr><td>34</td><td><code>01 00</code> v1/empty</td><td>null</td><td><code>34</code></td></tr><tr><td>null</td><td><code>01 00</code> v1/emp [...]
+Values in the two fields must be interpreted according to the following table:</p><table><thead><tr><th><code>value</code></th><th><code>typed_value</code></th><th>Meaning</th></tr></thead><tbody><tr><td>null</td><td>null</td><td>The value is missing; only valid for shredded object fields</td></tr><tr><td>non-null</td><td>null</td><td>The value is present and may be any type, including null</td></tr><tr><td>null</td><td>non-null</td><td>The value is present and is the shredded type</td>< [...]
+Writers must not produce data where both <code>value</code> and <code>typed_value</code> are non-null, unless the Variant value is an object.</p><p>If a Variant is missing in a context where a value is required, readers must return a Variant null (<code>00</code>): basic type 0 (primitive) and physical type 0 (null).
+For example, if a Variant is required (like <code>measurement</code> above) and both <code>value</code> and <code>typed_value</code> are null, the returned <code>value</code> must be <code>00</code> (Variant null).</p><h3 id=shredded-value-types>Shredded Value Types</h3><p>Shredded values must use the following Parquet types:</p><table><thead><tr><th>Variant Type</th><th>Parquet Physical Type</th><th>Parquet Logical Type</th></tr></thead><tbody><tr><td>boolean</td><td>BOOLEAN</td><td></t [...]
+If the value is an array, <code>value</code> must be null.</p><p>The list <code>element</code> must be a required group.
+The <code>element</code> group can contain <code>value</code> and <code>typed_value</code> fields.
+The element’s <code>value</code> field stores the element as Variant-encoded <code>binary</code> when the <code>typed_value</code> is not present or cannot represent it.
+The <code>typed_value</code> field may be omitted when not shredding elements as a specific type.
+The <code>value</code> field may be omitted when shredding elements as a specific type.
+However, at least one of the two fields must be present.</p><p>For example, a <code>tags</code> Variant may be shredded as a list of strings using the following definition:</p><pre tabindex=0><code>optional group tags (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value (LIST) { # must be optional to allow a null list
+ repeated group list {
+ required group element { # shredded element
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+}
+</code></pre><p>All elements of an array must be present (not missing) because the <code>array</code> Variant encoding does not allow missing elements.
+That is, either <code>typed_value</code> or <code>value</code> (but not both) must be non-null.
+Null elements must be encoded in <code>value</code> as Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>The series of <code>tags</code> arrays <code>["comedy", "drama"], ["horror", null], ["comedy", "drama", "romance"], null</code> would be stored as:</p><table><thead><tr><th>Array</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value...value</code></th><th><code>typed_value...typed_value</code></th></tr></thead><tbody><tr><td><code [...]
+If the value is not an object, <code>typed_value</code> must be null.
+Readers can assume that a value is not an object if <code>typed_value</code> is null and that <code>typed_value</code> field values are correct; that is, readers do not need to read the <code>value</code> column if <code>typed_value</code> fields satisfy the required fields.</p><p>Each shredded field in the <code>typed_value</code> group is represented as a required group that contains optional <code>value</code> and <code>typed_value</code> fields.
+The <code>value</code> field stores the value as Variant-encoded <code>binary</code> when the <code>typed_value</code> cannot represent the field.
+This layout enables readers to skip data based on the field statistics for <code>value</code> and <code>typed_value</code>.
+The <code>typed_value</code> field may be omitted when not shredding fields as a specific type.</p><p>The <code>value</code> column of a partially shredded object must never contain fields represented by the Parquet columns in <code>typed_value</code> (shredded fields).
+Readers may always assume that data is written correctly and that shredded fields in <code>typed_value</code> are not present in <code>value</code>.
+As a result, reads when a field is defined in both <code>value</code> and a <code>typed_value</code> shredded field may be inconsistent.</p><p>For example, a Variant <code>event</code> field may shred <code>event_type</code> (<code>string</code>) and <code>event_ts</code> (<code>timestamp</code>) columns using the following definition:</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value; # a variant, expected to be an object
+ optional group typed_value { # shredded fields for the variant object
+ required group event_type { # shredded field for event_type
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts { # shredded field for event_ts
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ }
+}
+</code></pre><p>The group for each named field must use repetition level <code>required</code>.</p><p>A field’s <code>value</code> and <code>typed_value</code> are set to null (missing) to indicate that the field does not exist in the variant.
+To encode a field that is present with a null value, the <code>value</code> must contain a Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>When both <code>value</code> and <code>typed_value</code> for a field are non-null, engines should fail.
+If engines choose to read in such cases, then the <code>typed_value</code> column must be used.
+Readers may always assume that data is written correctly and that only <code>value</code> or <code>typed_value</code> is defined.
+As a result, reads when both <code>value</code> and <code>typed_value</code> are defined may be inconsistent with optimized reads that require only one of the columns.</p><p>The table below shows how the series of objects in the first column would be stored:</p><table><thead><tr><th>Event object</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value.event_type.value</code></th><th><code>typed_value.event_type.typed_value</code></th><th><code>typed_value.eve [...]
+Readers must return an object when <code>typed_value</code> is non-null containing the shredded fields.</p><h2 id=nesting>Nesting</h2><p>The <code>typed_value</code> associated with any Variant <code>value</code> field can be any shredded type, as shown in the sections above.</p><p>For example, the <code>event</code> object above may also shred sub-fields as object (<code>location</code>) or array (<code>tags</code>).</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value {
+ required group event_type {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts {
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ required group location {
+ optional binary value;
+ optional group typed_value {
+ required group latitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ required group longitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ }
+ }
+ required group tags {
+ optional binary value;
+ optional group typed_value (LIST) {
+ repeated group list {
+ required group element {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+ }
+ }
+}
+</code></pre><h1 id=data-skipping>Data Skipping</h1><p>Statistics for <code>typed_value</code> columns can be used for file, row group, or page skipping when <code>value</code> is always null (missing).</p><p>When the corresponding <code>value</code> column is all nulls, all values must be the shredded <code>typed_value</code> field’s type.
+Because the type is known, comparisons with values of that type are valid.
+<code>IS NULL</code>/<code>IS NOT NULL</code> and <code>IS NAN</code>/<code>IS NOT NAN</code> filter results are also valid.</p><p>Comparisons with values of other types are not necessarily valid and data should not be skipped.</p><p>Casting behavior for Variant is delegated to processing engines.
+For example, the interpretation of a string as a timestamp may depend on the engine’s SQL session time zone.</p><h2 id=reconstructing-a-shredded-variant>Reconstructing a Shredded Variant</h2><p>It is possible to recover an unshredded Variant value using a recursive algorithm, where the initial call is to <code>construct_variant</code> with the top-level Variant group fields.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-siz [...]
+</span></span><span style=display:flex><span> <span style=color:#4e9a06>"""Constructs a Variant from value and typed_value"""</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>typed_value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>dict</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#000>object_fields</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#000>name</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> < [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>name</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#000>typed_value</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a partially shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>),</span> <span style=color:#4e9a06>"partially shredded value must be an object"</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>typed_value</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>keys</span><span style=color:#000;font-weight:700>()</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>isdisjoint</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#ce5c00;font [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># union the shredded fields and non-shredded fields</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>union</span><span style=color:#000;f [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded array</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded array must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#000>elements</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>[</span>
+</span></span><span style=display:flex><span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span> [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000>elem</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantArray</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elements</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded primitive</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded primitive must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># value is missing</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#204a87;font-weight:700>None</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>def</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>Any</span><span style=color:#000;font-weight:700>):</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>int</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantInteger</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>str</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantString</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>...</span>
+</span></span></code></pre></div><h2 id=backward-and-forward-compatibility>Backward and forward compatibility</h2><p>Shredding is an optional feature of Variant, and readers must continue to be able to read a group containing only <code>value</code> and <code>metadata</code> fields.</p><p>Engines that do not write shredded values must be able to read shredded values according to this spec or must fail.</p><p>Different files may contain conflicting shredding schemas.
+That is, files may contain different <code>typed_value</code> columns for the same Variant with incompatible types.
+It may not be possible to infer or specify a single shredded schema that would allow all Parquet files for a table to be read without reconstructing the value as a Variant.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-ad7f0d170ee709f2e58f9e27d7567421>4 -</h1><h1 id=variant-binary-encoding>Variant Binary Encoding</h1><p>A Variant represents a type that contains one of:</p><ul><li>Primitive: A type and corresponding value (e.g. INT, STRING)</li><li>Array: An orde [...]
+These represent a commonly supported subset of the <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>logical types</a> allowed by the Parquet format.</p><p>The Variant Binary Encoding allows representation of semi-structured data (e.g. JSON) in a form that can be efficiently queried by path.
+The design is intended to allow efficient access to nested data even in the presence of very wide or deep structures.</p><p>Another motivation for the representation is that (aside from metadata) each nested Variant value is contiguous and self-contained.
+For example, in a Variant containing an Array of Variant values, the representation of an inner Variant value, when paired with the metadata of the full variant, is itself a valid Variant.</p><p>This document describes the Variant Binary Encoding scheme.
+Variant fields can also be <em>shredded</em>.
+Shredding refers to extracting some elements of the variant into separate columns for more efficient extraction/filter pushdown.
+The <a href=/docs/file-format/types/variantshredding/>Variant Shredding specification</a> describes the details of shredding Variant values as typed Parquet columns.</p><h2 id=variant-in-parquet>Variant in Parquet</h2><p>A Variant value in Parquet is represented by a group with 2 fields, named <code>value</code> and <code>metadata</code>.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</c [...]
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group shredded_variant_name (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The <code>VARIANT</code> annotation places no additional restrictions on the repetition of Variant groups, but repetition may be restricted by containing types (such as <code>MAP</code> and <code>LIST</code>).
+The Variant group name is the name of the Variant column.</p><h2 id=metadata-encoding>Metadata encoding</h2><p>The encoded metadata always starts with a header byte.</p><pre tabindex=0><code> 7 6 5 4 3 0
+ +-------+---+---+---------------+
+header | | | | version |
+ +-------+---+---+---------------+
+ ^ ^
+ | +-- sorted_strings
+ +-- offset_size_minus_one
+</code></pre><p>The <code>version</code> is a 4-bit value that must always contain the value <code>1</code>.
+<code>sorted_strings</code> is a 1-bit value indicating whether dictionary strings are sorted and unique.
+<code>offset_size_minus_one</code> is a 2-bit value providing the number of bytes per dictionary size and offset field.
+The actual number of bytes, <code>offset_size</code>, is <code>offset_size_minus_one + 1</code>.</p><p>The entire metadata is encoded as the following diagram shows:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+metadata | header |
+ +-----------------------+
+ | |
+ : dictionary_size : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | | (`dictionary_size + 1` offsets)
+ +-----------------------+
+ | |
+ : bytes :
+ | |
+ +-----------------------+
+</code></pre><p>The metadata is encoded first with the <code>header</code> byte, then <code>dictionary_size</code> which is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the number of string values in the dictionary.
+Next, is an <code>offset</code> list, which contains <code>dictionary_size + 1</code> values.
+Each <code>offset</code> is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the starting byte offset of the i-th string in <code>bytes</code>.
+The first <code>offset</code> value will always be <code>0</code>, and the last <code>offset</code> value will always be the total length of <code>bytes</code>.
+The last part of the metadata is <code>bytes</code>, which stores all the string values in the dictionary.
+All string values must be UTF-8 encoded strings.</p><h3 id=metadata-encoding-grammar>Metadata encoding grammar</h3><p>The grammar for encoded metadata is as follows</p><pre tabindex=0><code>metadata: <header> <dictionary_size> <dictionary>
+header: 1 byte (<version> | <sorted_strings> << 4 | (<offset_size_minus_one> << 6))
+version: a 4-bit version ID. Currently, must always contain the value 1
+sorted_strings: a 1-bit value indicating whether metadata strings are sorted
+offset_size_minus_one: 2-bit value providing the number of bytes per dictionary size and offset field.
+dictionary_size: `offset_size` bytes. unsigned little-endian value indicating the number of strings in the dictionary
+dictionary: <offset>* <bytes>
+offset: `offset_size` bytes. unsigned little-endian value indicating the starting position of the ith string in `bytes`. The list should contain `dictionary_size + 1` values, where the last value is the total length of `bytes`.
+bytes: UTF-8 encoded dictionary string values
+</code></pre><p>Notes:</p><ul><li>Offsets are relative to the start of the <code>bytes</code> array.</li><li>The length of the ith string can be computed as <code>offset[i+1] - offset[i]</code>.</li><li>The offset of the first string is always equal to 0 and is therefore redundant. It is included in the spec to simplify in-memory-processing.</li><li><code>offset_size_minus_one</code> indicates the number of bytes per <code>dictionary_size</code> and <code>offset</code> entry. I.e. a valu [...]
+ +------------------------------------+------------+
+value | value_header | basic_type | <-- value_metadata
+ +------------------------------------+------------+
+ | |
+ : value_data : <-- 0 or more bytes
+ | |
+ +-------------------------------------------------+
+</code></pre><h3 id=basic-type>Basic Type</h3><p>The <code>basic_type</code> is 2-bit value that represents which basic type the Variant value is.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>basic types table</a> shows what each value represents.</p><h3 id=value-header>Value Header</h3><p>The <code>value_header</code> is a 6-bit value that contains more information about the type, and the format depends on the <code>basic_type</code>.</p><h4 id=value-header-for-primitive-type-basic_type0>Value Header for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>v [...]
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows what each value represents.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | primitive_header |
+ +-----------------------+
+</code></pre><h4 id=value-header-for-short-string-basic_type1>Value Header for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_header</code> is a 6-bit <code>short_string_header</code>.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | short_string_header |
+ +-----------------------+
+</code></pre><p>The <code>short_string_header</code> value is the length of the string.</p><h4 id=value-header-for-object-basic_type2>Value Header for Object (<code>basic_type</code>=2)</h4><p>When <code>basic_type</code> is <code>2</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, <code>field_id_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 4 3 2 1 0
+ +---+---+-------+-------+
+value_header | | | | |
+ +---+---+-------+-------+
+ ^ ^ ^
+ | | +-- field_offset_size_minus_one
+ | +-- field_id_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> and <code>field_id_size_minus_one</code> are 2-bit values that represent the number of bytes used to encode the field offsets and field ids.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code> and <code>field_id_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h4 id=value-header-for-array-basic_type3>Value Header for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 3 2 1 0
+ +-----------+---+-------+
+value_header | | | |
+ +-----------+---+-------+
+ ^ ^
+ | +-- field_offset_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> is a 2-bit value that represents the number of bytes used to encode the field offset.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h3 id=value-data>Value Data</h3><p>The <code>value_data</code> encoding format depends on the type specified by <code>value_metadata</code>.
+For some types, the <code>value_data</code> will be 0-bytes.</p><h4 id=value-data-for-primitive-type-basic_type0>Value Data for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>value_data</code> depends on the <code>primitive_header</code> value.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows the encoding format for each primitive type.</p><h4 id=value-data-for-short-string-basic_type1>Value Data for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_data</code> is the sequence of UTF-8 encoded bytes that represents the string.</p><h4 id=value-data-for-object-basic_type2>Value Data for Object (<code>basic_type</code>=2)</ [...]
+The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+object value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | | (`num_elements` field_ids)
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An object <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the object.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a list of <code>field_id</code> values.
+There are <code>num_elements</code> number of entries and each <code>field_id</code> is an unsigned little-endian value of <code>field_id_size</code> bytes.
+A <code>field_id</code> is an index into the dictionary in the metadata.
+The <code>field_id</code> list is followed by a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the end of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th key-value pair of the object, the key is the metadata dictionary entry indexed by the i-th <code>field_id</code>, and the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><p>The field ids and field offsets must be in lexicographical order of the corresponding field names in the metadata dictionary.
+However, the actual <code>value</code> entries do not need to be in any particular order.
+This implies that the <code>field_offset</code> values may not be monotonically increasing.
+For example, for the following object:</p><pre tabindex=0><code>{
+ "c": 3,
+ "b": 2,
+ "a": 1
+}
+</code></pre><p>The <code>field_id</code> list must be <code>[<id for key "a">, <id for key "b">, <id for key "c">]</code>, in lexicographical order.
+The <code>field_offset</code> list must be <code>[<offset for value 1>, <offset for value 2>, <offset for value 3>, <last offset>]</code>.
+The <code>value</code> list can be in any order.</p><h4 id=value-data-for-array-basic_type3>Value Data for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_data</code> encodes an array. The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+array value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An array <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the array.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the last byte of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th array entry, the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><h3 id=value-encoding-grammar>Value encoding grammar</h3><p>The grammar for an encoded value is:</p><pre tabindex=0><code>value: <value_metadata> <value_data>?
+value_metadata: 1 byte (<basic_type> | (<value_header> << 2))
+basic_type: ID from Basic Type table. <value_header> must be a corresponding variation
+value_header: <primitive_header> | <short_string_header> | <object_header> | <array_header>
+primitive_header: ID from Primitive Type table. <val> must be a corresponding variation of <primitive_val>
+short_string_header: unsigned string length in bytes from 0 to 63
+object_header: (is_large << 4 | field_id_size_minus_one << 2 | field_offset_size_minus_one)
+array_header: (is_large << 2 | field_offset_size_minus_one)
+value_data: <primitive_val> | <short_string_val> | <object_val> | <array_val>
+primitive_val: see table for binary representation
+short_string_val: UTF-8 encoded bytes
+object_val: <num_elements> <field_id>* <field_offset>* <fields>
+array_val: <num_elements> <field_offset>* <fields>
+num_elements: a 1 or 4 byte unsigned little-endian value (depending on is_large in <object_header>/<array_header>)
+field_id: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_id_size_minus_one in <object_header>), indexing into the dictionary
+field_offset: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_offset_size_minus_one in <object_header>/<array_header>), providing the offset in bytes within fields
+fields: <value>*
+</code></pre><p>Each <code>value_data</code> must correspond to the type defined by <code>value_metadata</code>. Boolean and null types do not have a corresponding <code>value_data</code>, since their type defines their value.</p><p>Each <code>array_val</code> and <code>object_val</code> must contain exactly <code>num_elements + 1</code> values for <code>field_offset</code>.
+The last entry is the offset that is one byte past the last field (i.e. the total size of all fields in bytes).
+All offsets are relative to the first byte of the first field in the object/array.</p><p><code>field_id_size_minus_one</code> and <code>field_offset_size_minus_one</code> indicate the number of bytes per field ID/offset.
+For example, a value of 0 indicates 1-byte IDs, 1 indicates 2-byte IDs, 2 indicates 3 byte IDs and 3 indicates 4-byte IDs.
+The <code>is_large</code> flag for arrays and objects is used to indicate whether the number of elements is indicated using a one or four byte value.
+When more than 255 elements are present, <code>is_large</code> must be set to true.
+It is valid for an implementation to use a larger value than necessary for any of these fields (e.g. <code>is_large</code> may be true for an object with less than 256 elements).</p><p>The “short string” basic type may be used as an optimization to fold string length into the type byte for strings less than 64 bytes.
+It is semantically identical to the “string” primitive type.</p><p>The Decimal type contains a scale, but no precision. The implied precision of a decimal value is <code>floor(log_10(val)) + 1</code>.</p><h2 id=encoding-types>Encoding types</h2><p><em>Variant basic types</em></p><table><thead><tr><th>Basic Type</th><th>ID</th><th>Description</th></tr></thead><tbody><tr><td>Primitive</td><td><code>0</code></td><td>One of the primitive types</td></tr><tr><td>Short string</td><t [...]
+For example, a user expression operating on a string value containing “hello” should behave the same, whether it is encoded with the short string optimization, or long string encoding.
+Similarly, user expressions operating on an <em>int8</em> value of 1 should behave the same as a decimal16 with scale 2 and unscaled value 100.</p><p><em>Decimal table</em></p><table><thead><tr><th>Decimal Precision</th><th>Decimal value type</th><th>Variant Physical Type</th></tr></thead><tbody><tr><td>1 <= precision <= 9</td><td>int32</td><td>decimal4</td></tr><tr><td>10 <= precision <= 18</td><td>int64</td><td>decimal8</td></tr><tr><td>19 <= precision <= 38</td><td>i [...]
+This includes the dictionary key string values, the “short string” values, and the “long string” values.</p><h2 id=object-field-id-order-and-uniqueness>Object field ID order and uniqueness</h2><p>For objects, field IDs and offsets must be listed in the order of the corresponding field names, sorted lexicographically (using unsigned byte ordering for UTF-8).
+Note that the field values themselves are not required to follow this order.
+As a result, offsets will not necessarily be listed in ascending order.
+The field values are not required to be in the same order as the field IDs, to enable flexibility when constructing Variant values.</p><p>An implementation may rely on this field ID order in searching for field names.
+E.g. a binary search on field IDs (combined with metadata lookups) may be used to find a field with a given name.</p><p>Field names are case-sensitive.
+Field names are required to be unique for each object.
+It is an error for an object to contain two fields with the same name, whether or not they have distinct dictionary IDs.</p><h2 id=versions-and-extensions>Versions and extensions</h2><p>An implementation is not expected to parse a Variant value whose metadata version is higher than the version supported by the implementation.
+However, new types may be added to the specification without incrementing the version ID.
+In such a situation, an implementation should be able to read the rest of the Variant value if desired.</p><h2 id=shredding>Shredding</h2><p>A single Variant object may have poor read performance when only a small subset of fields are needed.
+A better approach is to create separate columns for individual fields, referred to as shredding or subcolumnarization.
+<a href=/docs/file-format/types/variantshredding/>VariantShredding.md</a> describes the Variant shredding specification in Parquet.</p></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener hr [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/index.html b/output/_print/docs/index.html
index 39e73ed..ce65d00 100644
--- a/output/_print/docs/index.html
+++ b/output/_print/docs/index.html
@@ -2,8 +2,8 @@
The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><meta property="og:url" content="/docs/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Documentation"><meta property="og:description" content="Welcome to the documentation for Apache Parquet.
The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><meta property="og:locale" content="en"><meta property="og:type" content="website"><meta itemprop=name content="Documentation"><meta itemprop=description content="Welcome to the documentation for Apache Parquet.
The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><meta itemprop=dateModified content="2025-10-28T06:46:33-04:00"><meta itemprop=wordCount content="36"><meta name=twitter:card content="summary"><meta name=twitter:title content="Documentation"><meta name=twitter:description content="Welcome to the documentation for Apache Parquet.
-The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c [...]
-<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/>Return to the regular view of this page</a>.</p></div><h1 class=title>Documentation</h1><ul><li>1: <a href=#pg-6e17e09fffc1050f46600282def85180>Overview</a></li><ul><li>1.1: <a href=#pg-2e5324b574579d3eb78ce816487d9e18>Motivation</a></li></ul><li>2: <a href=#pg-fcf6186c3c17ed7f2e8577136109a520>Concepts</a></li><ul></ul><li>3: <a href=#pg-52ee54aeff1ffc82031ec74e9a626eba>File Format</a></li><ul><li>3.1: <a [...]
+The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de [...]
+<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/>Return to the regular view of this page</a>.</p></div><h1 class=title>Documentation</h1><ul><li>1: <a href=#pg-6e17e09fffc1050f46600282def85180>Overview</a></li><ul><li>1.1: <a href=#pg-2e5324b574579d3eb78ce816487d9e18>Motivation</a></li></ul><li>2: <a href=#pg-fcf6186c3c17ed7f2e8577136109a520>Concepts</a></li><ul></ul><li>3: <a href=#pg-52ee54aeff1ffc82031ec74e9a626eba>File Format</a></li><ul><li>3.1: <a [...]
The current implementation status of various features can be found in the <a href=/docs/file-format/implementationstatus/>implementation status</a> page.</p></div></div><div class=td-content><h1 id=pg-6e17e09fffc1050f46600282def85180>1 - Overview</h1><div class=lead>All about Parquet.</div><p>Apache Parquet is an open source, column-oriented data file format designed for efficient data storage and retrieval.
It provides high performance compression and encoding schemes to handle complex data in bulk and is supported in many programming languages and analytics tools.</p><h3 id=parquet-format-specification>parquet-format (Specification)</h3><p>The <a href=https://github.com/apache/parquet-format>parquet-format</a> repository hosts the official specification of the Parquet file format, defining how data is structured and stored. This specification, along with the <a href=https://github.com/apac [...]
unchanged for describing this file format. The file format is
@@ -38,7 +38,70 @@ start locations. More details on what is contained in the metadata can be found
in the Thrift definition.</p><p>File metadata is written after the data to allow for single pass writing.</p><p>Readers are expected to first read the file metadata to find all the column
chunks they are interested in. The columns chunks should then be read sequentially.</p><p>The format is explicitly designed to separate the metadata from the data. This
allows splitting columns into multiple files, as well as having a single metadata
-file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>3.1 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
+file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-a8f87c969a3621c87488ade11fb64f28>3.1 -</h1><h1 id=binary-protocol-extensions>Binary Protocol Extensions</h1><p>The extension mechanism of the <code>binary</code> Thrift field-id <code>32767</code> has some desirable properties:</p><ul><li>Existing readers will ignore these extensions without any modifications</li><li>Exist [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(M) encoded size of the extension
+M bytes | extension bytes
+1 byte | \0 (thrift stop field)
+</code></pre><p>The choice to reserve only one field-id has an additional (and frankly unintended) property. It creates scarcity in the extension space and disincentivizes vendors from keeping their extensions private. As a vendor having an extension means one cannot use it in tandem with other extensions from other vendors even if such extensions are publicly known. The easiest path of interoperability and ability to further experiment is to push an extension through standardization and [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(K+28) encoded size of the extension
+K bytes | Flatbuffers representation (v0) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+16 bytes | some-UUID
+1 byte | \0 (thrift stop field)
+4 bytes | PAR1
+</code></pre><p>some-UUID is some UUID picked for this extension and it is used throughout (possibly internal) experimentation. It is put at the end to allow detection of the extension when parsed in reverse. The little-endian sizes and crc32s are also to the end to facilitate efficient parsing the footer in reverse without requiring parsing the Thrift compact protocol that precedes it.</p><p>At some point the experiments conclude and the extension shared publicly with the community. The [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(K+28) encoded size of the extension
+K bytes | Flatbuffers representation (v1) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+16 bytes | some-other-UUID
+1 byte | \0 (thrift stop field)
+4 bytes | PAR1
+</code></pre><p>After the migration period, the end of the Parquet file may look like this:</p><pre><code>K bytes | Flatbuffers representation (v1) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+4 bytes | PAR3
+</code></pre><p>In this example, we see several design decisions for the extension at play:</p><ul><li>There is a new some-other-UUID for the accepted change to the standard and now the Thrift <code>FileMetaData</code> cannot be extended itself.</li><li>The length of the footer and the crc32 of the length itself, guarantees that new readers will not overshoot reading bytes in case of corrupt bits in these critical 8 bytes of the file.</li><li>The crc32 of the flatbuffer representation en [...]
+ | | Column b (new encoding)
+ | | Column c (new encoding)
+R bytes | Row Group | Column a
+ | 0 | Column d
+ | | Column b (old encoding)
+ | | Column c (old encoding)
+ | | FileMetaData
+ | | ColumnMetaData: a
+ | | ColumnMetaData: b
+F bytes | | <extension-blob with offsets to new encoding>
+ | | ColumnMetaData: c
+ | | <extension-blob with offsets to new encoding>
+ | | ColumnMetaData: d
+4 bytes | PAR1
+</code></pre><p>The custom reader is compiled with thrift IDL with a binary for field with id 32767. This is done to become extension aware and inspect the extension bytes looking for the UUID disambiguator. If that’s found it decodes the offsets from the rest of the bytes and reads the region of the file containing the new encoding.</p><p>If/when the encoding is ratified, it is added to the official specification as an additional type in <code>Encodings</code> at which point the extensi [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>while</span> <span style=color:#000;font-weight:700>(</span><span style=color:#204a87>true</span><span style=color:#000;font-weight:700>)</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>uint8_t</span> <span style=color:#000>c</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700>&</span> <span style=color:#0000cf;font-weight:700>0x7F</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700><</span> <span style=color:#0000cf;font-weight:700>0x80</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>out</span><span style=color:#ce5c00;font-weight:700>-></span><span style=co [...]
+</span></span><span style=display:flex><span> <span style=color:#000>out</span><span style=color:#ce5c00;font-weight:700>-></span><span style=color:#000>push_back</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>c</span> <span style=color:#ce5c00;font-weight:700>+</span> <span style=color:#0000cf;font-weight:700>0x80</span><span style=color:#000;font-weight:700>);</span>
+</span></span><span style=display:flex><span> <span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700>>>=</span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>};</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#000>std</span><span style=color:#ce5c00;font-weight:700>::</span><span style=color:#000>string</span> <span style=color:#000>AppendExtension</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>std</span><span style=color:#ce5c00;font-weight:700>::</span><span style=color:#000>string</span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;fon [...]
+</span></span><span style=display:flex><span> <span style=color:#000>assert</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>back</span><span style=color:#000;font-weight:700>()</span> <span style=color:#ce5c00;font-weight:700>==</span> <span style=color:#4e9a06>'\x00'</span><span style=color:#000;font-weight:700>);</span> <span style=color:#8f5902;font-style:italic>// [...]
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>back</span><span style=color:#000;font-weight:700>()</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#4e9a06>'\x08'</span><span style=color:#000;font-weight:700>;</span> <span style=color:#8f5902;font-style:italic>// replace stop field with [...]
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AppendUleb</span><span style=color:#000;font-weight:700>(</span><span style=color:#0000cf;font-weight:700>32767</span><span style=color:#000;font-weight:700>,</span> <span style=color:#ce5c00;font-weight:700>&</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>);</span> <span style=color:#8f5902;font-style:italic>// field-id
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AppendUleb</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>ext</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>size</span><span style=color:#000;font-weight:700>(),</span> <span style=color:#ce5c00;font-weight:700>&</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>);</span>
+</span></span><span style=display:flex><span> <span style=color:#000>thrift</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#000>ext</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#000>thrift</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#4e9a06>'\x00'</span><span style=color:#000;font-weight:700>;</span> <span style=color:#8f5902;font-style:italic>// add the stop field
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
+</span></span></code></pre></div></div><div class=td-content style=page-break-before:always><h1 id=pg-de128b03a7277c17c6dec2cef97af32d>3.2 - Configurations</h1><h3 id=row-group-size>Row Group Size</h3><p>Larger row groups allow for larger column chunks which makes it
possible to do larger sequential IO. Larger groups also require more buffering in
the write path (or a two pass write). We recommend large row groups (512MB - 1GB).
Since an entire row group might need to be read, we want it to completely fit on
@@ -48,12 +111,12 @@ per HDFS file.</p><h3 id=data-page--size>Data Page Size</h3><p>Data pages should
allow for more fine grained reading (e.g. single row lookup). Larger page sizes
incur less space overhead (less page headers) and potentially less parsing overhead
(processing headers). Note: for sequential scans, it is not expected to read a page
-at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-6762d78210357c1df172dfddcc6fd307>3.2 - Extensibility</h1><p>There are many places in the format for compatible extensions:</p><ul><li>File Version: The file metadata contains a version.</li><li>Encodings: Encodings are specified by enum and more can be added in the future.</li><li>Page types: Additional page types can be added and safely skipped.</ [...]
+at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-6762d78210357c1df172dfddcc6fd307>3.3 - Extensibility</h1><p>There are many places in the format for compatible extensions:</p><ul><li>File Version: The file metadata contains a version.</li><li>Encodings: Encodings are specified by enum and more can be added in the future.</li><li>Page types: Additional page types can be added and safely skipped.</ [...]
definition of these structures is given in the Parquet
<a href=https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift>Thrift definition</a>.</p><h2 id=file-metadata>File metadata</h2><p>In the diagram below, file metadata is described by the <code>FileMetaData</code>
structure. This file metadata provides offset and size information useful
when navigating the Parquet file.</p><p><img src=/images/FileMetaData.svg alt="Parquet Metadata format"></p><h2 id=page-header>Page header</h2><p>Page header metadata (<code>PageHeader</code> and children in the diagram) is stored
-in-line with the page data, and is used in the reading and decoding of data.</p><p><img src=/images/PageHeader.svg alt="Parquet PageHeader format"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>3.4 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
+in-line with the page data, and is used in the reading and decoding of data.</p><p><img src=/images/PageHeader.svg alt="Parquet PageHeader format"></p></div><div class=td-content style=page-break-before:always><h1 id=pg-140f2d6d5609da0af9dcaa92233086b5>3.5 - Types</h1><p>The types supported by the file format are intended to be as minimal as possible,
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
32-bit ints with an efficient encoding. This reduces the complexity of implementing
@@ -65,19 +128,828 @@ readers and writers for the format. The types are:</p><pre tabindex=0><code> -
- DOUBLE: IEEE 64-bit floating point values
- BYTE_ARRAY: arbitrarily long byte arrays
- FIXED_LEN_BYTE_ARRAY: fixed length byte arrays
-</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-967cf8be84f5efbfd2e57475707b79b1>3.4.1 - Logical Types</h1><p>Logical types are used to extend the types that parquet can be used to store,
+</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-95b2ce86b5e36722017e238af2bd0fd8>3.5.1 -</h1><h1 id=geospatial-definitions>Geospatial Definitions</h1><p>This document contains the specification of geospatial types and statistics.</p><h1 id=background>Background</h1><p>The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and
+Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM,
+XYZM) are defined by <a href="https://portal.ogc.org/files/?artifact_id=25355">OpenGIS Implementation Specification for Geographic
+information - Simple feature access - Part 1: Common architecture</a>,
+from <a href=https://www.ogc.org/standard/sfa/>OGC(Open Geospatial Consortium)</a>.</p><p>The version of the OGC standard first used here is 1.2.1, but future versions
+may also be used if the WKB representation remains wire-compatible.</p><h2 id=coordinate-reference-system>Coordinate Reference System</h2><p>Coordinate Reference System (CRS) is a mapping of how coordinates refer to
+locations on Earth.</p><p>The default CRS <code>OGC:CRS84</code> means that the geospatial features must be stored
+in the order of longitude/latitude based on the WGS84 datum.</p><p>Custom CRS can be specified by a string value. It is recommended to use an
+identifier-based approach like <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>.</p><p>For geographic CRS, longitudes are bound by [-180, 180] and latitudes are bound
+by [-90, 90].</p><h2 id=edge-interpolation-algorithm>Edge Interpolation Algorithm</h2><p>An algorithm for interpolating edges, and is one of the following values:</p><ul><li><code>spherical</code>: edges are interpolated as geodesics on a sphere.</li><li><code>vincenty</code>: <a href=https://en.wikipedia.org/wiki/Vincenty%27s_formulae>https://en.wikipedia.org/wiki/Vincenty%27s_formulae</a></li><li><code>thomas</code>: Thomas, Paul D. Spheroidal geodesics, reference systems, & local geom [...]
+logical types to store statistics of a column chunk. It is an optional field in
+the <code>ColumnMetaData</code> and contains <a href=/docs/file-format/types/geospatial/#bounding-box>Bounding Box</a> and <a href=/docs/file-format/types/geospatial/#geospatial-types>Geospatial
+Types</a> that are described below in detail.</p><h2 id=bounding-box>Bounding Box</h2><p>A geospatial instance has at least two coordinate dimensions: X and Y for 2D
+coordinates of each point. Please note that X is longitude/easting and Y is
+latitude/northing. A geospatial instance can optionally have Z and/or M values
+associated with each point.</p><p>The Z values introduce the third dimension coordinate. Usually they are used to
+indicate the height, or elevation.</p><p>M values are an opportunity for a geospatial instance to track a value in a
+fourth dimension. These values can be used as a linear reference value (e.g.,
+highway milepost value), a timestamp, or some other value as defined by the CRS.</p><p>Bounding box is defined as the thrift struct below in the representation of
+min/max value pair of coordinates from each axis. Note that X and Y Values are
+always present. Z and M are omitted for 2D geospatial instances.</p><p>When calculating a bounding box, null or NaN values in a coordinate
+dimension are skipped. For example, <code>POINT (1 NaN)</code> contributes a value to X
+but no values to Y, Z, or M dimension of the bounding box. If a dimension has
+only null or NaN values, that dimension is omitted from the bounding box. If
+either the X or Y dimension is missing, then the bounding box itself is not
+produced.</p><p>For the X values only, xmin may be greater than xmax. In this case, an object
+in this bounding box may match if it contains an X such that <code>x >= xmin</code> OR
+<code>x <= xmax</code>. This wraparound occurs only when the corresponding bounding box
+crosses the antimeridian line. In geographic terminology, the concepts of <code>xmin</code>,
+<code>xmax</code>, <code>ymin</code>, and <code>ymax</code> are also known as <code>westernmost</code>, <code>easternmost</code>,
+<code>southernmost</code> and <code>northernmost</code>, respectively.</p><p>For <code>GEOGRAPHY</code> types, X and Y values are restricted to the canonical ranges of
+[-180, 180] for X and [-90, 90] for Y.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-thrift data-lang=thrift><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>BoundingBox</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000;font-weight:700>{</span><s [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>2</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>3</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>4</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>5</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>6</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>8</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#000;font-weight:700>}</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span></code></pre></div><h2 id=geospatial-types>Geospatial Types</h2><p>A list of geospatial types from all instances in the <code>GEOMETRY</code> or <code>GEOGRAPHY</code>
+column, or an empty list if they are not known.</p><p>This is borrowed from <a href="https://github.com/opengeospatial/geoparquet/blob/v1.1.0/format-specs/geoparquet.md?plain=1#L159">geometry_types of GeoParquet</a> except that
+values in the list are <a href=https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary>WKB (ISO-variant) integer codes</a>.
+Table below shows the most common geospatial types and their codes:</p><table><thead><tr><th style=text-align:left>Type</th><th style=text-align:left>XY</th><th style=text-align:left>XYZ</th><th style=text-align:left>XYM</th><th style=text-align:center>XYZM</th></tr></thead><tbody><tr><td style=text-align:left>Point</td><td style=text-align:left>0001</td><td style=text-align:left>1001</td><td style=text-align:left>2001</td><td style=text-align:center>3001</td></tr><tr><td style=text-alig [...]
+responsible for serializing and deserializing the CRS, respectively.</p><p>As a convention to maximize the interoperability, custom CRS values can be
+specified by a string of the format <code>type:identifier</code>, where <code>type</code> is one of
+the following values:</p><ul><li><code>srid</code>: <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>, <code>identifier</code> is the SRID itself.</li><li><code>projjson</code>: <a href=https://proj.org/en/stable/specifications/projjson.html>PROJJSON</a>, <code>identifier</code> is the name of a table property or a file property where the projjson string is stored.</li></ul><h1 id=coordinate-axis-order>Coordinate axis order</h1><p> [...]
+follows the de facto standard for axis order in WKB and is therefore always
+(x, y) where x is easting or longitude and y is northing or latitude. This
+ordering explicitly overrides the axis order as specified in the CRS.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-967cf8be84f5efbfd2e57475707b79b1>3.5.2 -</h1><h1 id=parquet-logical-type-definitions>Parquet Logical Type Definitions</h1><p>Logical types are used to extend the types that parquet can be used to store,
by specifying how the primitive types should be interpreted. This keeps the set
of primitive types to a minimum and reuses parquet’s efficient encodings. For
-example, strings are stored as byte arrays (binary) with a UTF8 annotation.
-These annotations define how to further decode and interpret the data.
-Annotations are stored as <code>LogicalType</code> fields in the file metadata and are
-documented in <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>LogicalTypes.md</a></p></div><div class=td-content style=page-break-before:always><h1 id=pg-0d8efcf7d4835e2ce724ae57b02d4318>3.5 - Nested Encoding</h1><p>To encode nested columns, Parquet uses the Dremel encoding with definition and
+example, strings are stored with the primitive type <code>BYTE_ARRAY</code> with a <code>STRING</code>
+annotation.</p><p>This file contains the specification for all logical types.</p><h3 id=metadata>Metadata</h3><p>The parquet format’s <code>LogicalType</code> stores the type annotation. The annotation
+may require additional metadata fields, as well as rules for those fields.</p><p>There is an older representation of the logical type annotations called <code>ConvertedType</code>.
+To support backward compatibility with old files, readers should interpret <code>LogicalTypes</code>
+in the same way as <code>ConvertedType</code>, and writers should populate <code>ConvertedType</code> in the metadata
+according to well defined conversion rules.</p><h3 id=compatibility>Compatibility</h3><p>The Thrift definition of the metadata has two fields for logical types: <code>ConvertedType</code> and <code>LogicalType</code>.
+<code>ConvertedType</code> is an enum of all available annotations. Since Thrift enums can’t have additional type parameters,
+it is cumbersome to define additional type parameters, like decimal scale and precision
+(which are additional 32 bit integer fields on SchemaElement, and are relevant only for decimals) or time unit
+and UTC adjustment flag for Timestamp types. To overcome this problem, a new logical type representation was introduced into
+the metadata to replace <code>ConvertedType</code>: <code>LogicalType</code>. The new representation is a union of structs of logical types,
+this way allowing more flexible API, logical types can have type parameters.</p><p><code>ConvertedType</code> is deprecated. However, to maintain compatibility with old writers,
+Parquet readers should be able to read and interpret <code>ConvertedType</code> annotations
+in case <code>LogicalType</code> annotations are not present. Parquet writers must always write
+<code>LogicalType</code> annotations where applicable, but must also write the corresponding
+<code>ConvertedType</code> annotations (if any) to maintain compatibility with old readers.</p><p>Compatibility considerations are mentioned for each annotation in the corresponding section.</p><h2 id=string-types>String Types</h2><h3 id=string>STRING</h3><p><code>STRING</code> may only be used to annotate the <code>BYTE_ARRAY</code> primitive type and indicates
+that the byte array should be interpreted as a UTF-8 encoded character string.</p><p>The sort order used for <code>STRING</code> strings is unsigned byte-wise comparison.</p><p><em>Compatibility</em></p><p><code>STRING</code> corresponds to <code>UTF8</code> ConvertedType.</p><h3 id=enum>ENUM</h3><p><code>ENUM</code> annotates the <code>BYTE_ARRAY</code> primitive type and indicates that the value
+was converted from an enumerated type in another data model (e.g. Thrift, Avro, Protobuf).
+Applications using a data model lacking a native enum type should interpret <code>ENUM</code>
+annotated field as a UTF-8 encoded string.</p><p>The sort order used for <code>ENUM</code> values is unsigned byte-wise comparison.</p><h3 id=uuid>UUID</h3><p><code>UUID</code> annotates a 16-byte <code>FIXED_LEN_BYTE_ARRAY</code> primitive type. The value is
+encoded using big-endian, so that <code>00112233-4455-6677-8899-aabbccddeeff</code> is encoded
+as the bytes <code>00 11 22 33 44 55 66 77 88 99 aa bb cc dd ee ff</code>
+(This example is from <a href=https://en.wikipedia.org/wiki/Universally_unique_identifier>wikipedia’s UUID page</a>).</p><p>The sort order used for <code>UUID</code> values is unsigned byte-wise comparison.</p><h2 id=numeric-types>Numeric Types</h2><h3 id=signed-integers>Signed Integers</h3><p><code>INT</code> annotation can be used to specify the maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+For signed integers, the second parameter should be <code>true</code>,
+for example, a signed integer with bit width of 8 is defined as <code>INT(8, true)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, true)</code>, <code>INT(16, true)</code>, and <code>INT(32, true)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, true)</code> must annotate an <code>int64</code> primitive type. <code>INT(32, true)</code> and <code>INT(64, true)</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p>The sort order used for signed integer types is signed.</p><h3 id=unsigned-integers>Unsigned Integers</h3><p><code>INT</code> annotation can be used to specify unsigned integer types,
+along with a maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+In case of unsigned integers, the second parameter should be <code>false</code>,
+for example, an unsigned integer with bit width of 8 is defined as <code>INT(8, false)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, false)</code>, <code>INT(16, false)</code>, and <code>INT(32, false)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, false)</code> must annotate an <code>int64</code> primitive type.</p><p>The sort order used for unsigned integer types is unsigned.</p><h3 id=deprecated-integer-convertedtype>Deprecated integer ConvertedType</h3><p><code>INT_8</code>, <code>INT_16</code>, <code>INT_32</code>, and <code>INT_64</code> annotations can be also used to specify
+signed integers with 8, 16, 32, or 64 bit width.</p><p><code>INT_8</code>, <code>INT_16</code>, and <code>INT_32</code> must annotate an <code>int32</code> primitive type and
+<code>INT_64</code> must annotate an <code>int64</code> primitive type. <code>INT_32</code> and <code>INT_64</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p><code>UINT_8</code>, <code>UINT_16</code>, <code>UINT_32</code>, and <code>UINT_64</code> annotations can be also used to specify
+unsigned integers with 8, 16, 32, or 64 bit width.</p><p><code>UINT_8</code>, <code>UINT_16</code>, and <code>UINT_32</code> must annotate an <code>int32</code> primitive type and
+<code>UINT_64</code> must annotate an <code>int64</code> primitive type.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>INT_8</td><td>IntType (bitWidth = 8, isSigned = true)</td></tr><tr><td>INT_16</td><td>IntType (bitWidth = 16, isSigned = true)</td></tr><tr><td>INT_32</td><td>IntType (bitWidth = 32, isSigned = true)</td></tr><tr><td>INT_64</td><td>IntType (bitWidth = 64, isSigned = true)</td></tr><tr><td [...]
+the form <code>unscaledValue * 10^(-scale)</code>.</p><p>The primitive type stores an unscaled integer value. For <code>BYTE_ARRAY</code> and
+<code>FIXED_LEN_BYTE_ARRAY</code>, the unscaled number must be encoded as two’s complement using
+big-endian byte order (the most significant byte is the zeroth element). The
+scale stores the number of digits of that value that are to the right of the
+decimal point, and the precision stores the maximum number of digits supported
+in the unscaled value.</p><p>If not specified, the scale is 0. Scale must be zero or a positive integer less
+than or equal to the precision. Precision is required and must be a non-zero positive
+integer. A precision too large for the underlying type (see below) is an error.</p><p><code>DECIMAL</code> can be used to annotate the following types:</p><ul><li><code>int32</code>: for 1 <= precision <= 9</li><li><code>int64</code>: for 1 <= precision <= 18; precision < 10 will produce a
+warning</li><li><code>fixed_len_byte_array</code>: precision is limited by the array size. Length <code>n</code>
+can store <= <code>floor(log_10(2^(8*n - 1) - 1))</code> base-10 digits</li><li><code>byte_array</code>: <code>precision</code> is not limited, but is required. The minimum number of
+bytes to store the unscaled value should be used.</li></ul><p>The sort order used for <code>DECIMAL</code> values is signed comparison of the represented
+value.</p><p>If the column uses <code>int32</code> or <code>int64</code> physical types, then signed comparison of
+the integer values produces the correct ordering. If the physical type is
+fixed, then the correct ordering can be produced by flipping the
+most-significant bit in the first byte and then using unsigned byte-wise
+comparison.</p><p><em>Compatibility</em></p><p>To support compatibility with older readers, implementations of parquet-format should
+write <code>DecimalType</code> precision and scale into the corresponding SchemaElement field in metadata.</p><h3 id=float16>FLOAT16</h3><p>The <code>FLOAT16</code> annotation represents half-precision floating-point numbers in the 2-byte IEEE little-endian format.</p><p>Used in contexts where precision is traded off for smaller footprint and potentially better performance.</p><p>The primitive type is a 2-byte <code>FIXED_LEN_BYTE_ARRAY</code>.</p><p>The sort order for <code>FLOAT16</cod [...]
+annotate an <code>int32</code> that stores the number of days from the Unix epoch, 1
+January 1970.</p><p>The sort order used for <code>DATE</code> is signed.</p><h3 id=time>TIME</h3><p><code>TIME</code> is used for a logical time type without a date with millisecond or microsecond precision.
+The type has two type parameters: UTC adjustment (<code>true</code> or <code>false</code>)
+and unit (<code>MILLIS</code> or <code>MICROS</code>, <code>NANOS</code>).</p><p><code>TIME</code> with unit <code>MILLIS</code> is used for millisecond precision.
+It must annotate an <code>int32</code> that stores the number of
+milliseconds after midnight.</p><p><code>TIME</code> with unit <code>MICROS</code> is used for microsecond precision.
+It must annotate an <code>int64</code> that stores the number of
+microseconds after midnight.</p><p><code>TIME</code> with unit <code>NANOS</code> is used for nanosecond precision.
+It must annotate an <code>int64</code> that stores the number of
+nanoseconds after midnight.</p><p>The sort order used for <code>TIME</code> is signed.</p><h4 id=deprecated-time-convertedtype>Deprecated time ConvertedType</h4><p><code>TIME_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int32</code>.</p><p><code>TIME_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MICROS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local time semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local time with legacy <code>TIME_MICROS</code> and <code>TIME_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local time with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIME_MILLIS</td><td>TimeType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIME_MICROS</td><td>TimeType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimeType</td><td rowspan=3>isAdj [...]
+<code>int64</code> number that can be decoded into year, month, day, hour, minute, second
+and subsecond fields using calculations detailed below. Please note that a value
+defined this way does not necessarily correspond to a single instant on the
+time-line and such interpretations are allowed on purpose.</p><p>The <code>TIMESTAMP</code> type has two type parameters:</p><ul><li><code>isAdjustedToUTC</code> must be either <code>true</code> or <code>false</code>.</li><li><code>unit</code> must be one of <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>. This list is subject
+to potential expansion in the future. Upon reading, unknown <code>unit</code>-s must
+be handled as unsupported features (rather than as errors in the data files).</li></ul><h4 id=instant-semantics-timestamps-normalized-to-utc>Instant semantics (timestamps normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=true</code> is defined as the number of
+milliseconds, microseconds or nanoseconds (depending on the <code>unit</code>
+parameter being <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>, respectively) elapsed since the
+Unix epoch, 1970-01-01 00:00:00 UTC. Each such value unambiguously identifies a
+single instant on the time-line.</p><p>For example, in a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>, the
+number 172800000 corresponds to 1970-01-03 00:00:00 UTC, because it is equal to
+2 * 24 * 60 * 60 * 1000, therefore it is exactly two days from the reference
+point, the Unix epoch. In Java, this calculation can be achieved by calling
+<code>Instant.ofEpochMilli(172800000)</code>.</p><p>As a slightly more complicated example, if one wants to store 1970-01-03
+00:00:00 (UTC+01:00) as a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>,
+first the time zone offset has to be dealt with. By normalizing the timestamp to
+UTC, we calculate what time in UTC corresponds to the same instant: 1970-01-02
+23:00:00 UTC. This is 1 day and 23 hours after the epoch, therefore it can be
+encoded as the number (24 + 23) * 60 * 60 * 1000 = 169200000.</p><p>Please note that time zone information gets lost in this process. Upon reading a
+value back, we can only reconstruct the instant, but not the original
+representation. In practice, such timestamps are typically displayed to users in
+their local time zones, therefore they may be displayed differently depending on
+the execution environment.</p><h4 id=local-semantics-timestamps-not-normalized-to-utc>Local semantics (timestamps not normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=false</code> represents year, month, day, hour,
+minute, second and subsecond fields in a local timezone, <em>regardless of what
+specific time zone is considered local</em>. This means that such timestamps should
+always be displayed the same way, regardless of the local time zone in effect.
+On the other hand, without additional information such as an offset or
+time-zone, these values do not identify instants on the time-line unambiguously,
+because the corresponding instants would depend on the local time zone.</p><p>Using a single number to represent a local timestamp is a lot less intuitive
+than for instants. One must use a local timestamp as the reference point and
+calculate the elapsed time between the actual timestamp and the reference point.
+The problem is that the result may depend on the local time zone, for example
+because there may have been a daylight saving time change between the two
+timestamps.</p><p>The solution to this problem is to use a simplification that makes the result
+easy to calculate and independent of the timezone. Treating every day as
+consisting of exactly 86400 seconds and ignoring DST changes altogether allows
+us to unambiguously represent a local timestamp as a difference from a reference
+local timestamp. We define the reference local timestamp to be 1970-01-01
+00:00:00 (note the lack of UTC at the end, as this is not an instant). This way
+the encoding of local timestamp values becomes very similar to the encoding of
+instant values. For example, in a <code>TIMESTAMP(isAdjustedToUTC=false, unit=MILLIS)</code>, the number 172800000 corresponds to 1970-01-03 00:00:00
+(note the lack of UTC at the end), because it is exactly two days from the
+reference point (172800000 = 2 * 24 * 60 * 60 * 1000).</p><p>Another way to get to the same definition is to treat the local timestamp value
+<em>as if</em> it were in UTC and store it as an instant. For example, if we treat the
+local timestamp 1970-01-03 00:00:00 <em>as if</em> it were the instant 1970-01-03
+00:00:00 UTC, we can store it as 172800000. When reading 172800000 back, we can
+reconstruct the instant 1970-01-03 00:00:00 UTC and convert it to a local
+timestamp <em>as if</em> we were in the UTC time zone, resulting in 1970-01-03
+00:00:00. In Java, this can be achieved by calling
+<code>LocalDateTime.ofEpochSecond(172800, 0, ZoneOffset.UTC)</code>.</p><p>Please note that while from a practical point of view this second definition is
+equivalent to the first one, from a theoretical point of view only the first
+definition can be considered correct, the second one just “incidentally” leads
+to the same results. Nevertheless, this second definition is worth mentioning as
+well, because it is relatively widespread and it can lead to confusion,
+especially due to its usage of UTC in the calculations. One can stumble upon
+code, comments and specifications ambiguously stating that a timestamp “is
+stored in UTC”. In some contexts, it means that it is <em>normalized</em> to UTC and
+acts as an instant. In some other contexts though, it means the exact opposite,
+namely that the timestamp is stored <em>as if</em> it were in UTC and acts as a
+local timestamp in reality.</p><h4 id=common-considerations>Common considerations</h4><p>Every possible <code>int64</code> number represents a valid timestamp, but depending on the
+precision, the corresponding year may be outside of the practical everyday
+limits and implementations may choose to only support a limited range.</p><p>On the other hand, not every combination of year, month, day, hour, minute,
+second and subsecond values can be encoded into an <code>int64</code>. Most notably:</p><ul><li>An arbitrary combination of timestamp fields can not be encoded as a single
+number if the values for some of the fields are outside of their normal range
+(where the “normal range” corresponds to everyday usage). For example, neither
+of the following can be represented in a timestamp:<ul><li>hour = -1</li><li>hour = 25</li><li>minute = 61</li><li>month = 13</li><li>day = 29, month = 2, year = any non-leap year</li></ul></li><li>Due to the range of the <code>int64</code> type, timestamps using the <code>NANOS</code> unit
+can only represent values between 1677-09-21 00:12:43 and 2262-04-11 23:47:16.
+Values outside of this range can not be represented with the <code>NANOS</code>
+unit. (Other precisions have similar limits but those are outside of the
+domain for practical everyday usage.)</li></ul><p>The sort order used for <code>TIMESTAMP</code> is signed.</p><h4 id=deprecated-timestamp-convertedtype>Deprecated timestamp ConvertedType</h4><p><code>TIMESTAMP_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p><code>TIMESTAMP_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MICROS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local timestamp semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local timestamps with legacy <code>TIMESTAMP_MICROS</code> and <code>TIMESTAMP_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local timestamps with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIMESTAMP_MILLIS</td><td>TimestampType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIMESTAMP_MICROS</td><td>TimestampType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimestampTyp [...]
+<code>fixed_len_byte_array</code> of length 12. This array stores three little-endian
+unsigned integers that represent durations at different granularities of time.
+The first stores a number in months, the second stores a number in days, and
+the third stores a number in milliseconds. This representation is independent
+of any particular timezone or date.</p><p>Each component in this representation is independent of the others. For
+example, there is no requirement that a large number of days should be
+expressed as a mix of months and days because there is not a constant
+conversion from days to months.</p><p>The sort order used for <code>INTERVAL</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=embedded-types>Embedded Types</h2><p>Embedded types do not have type-specific orderings.</p><h3 id=json>JSON</h3><p><code>JSON</code> is used for an embedded JSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as a UTF-8 encoded character
+string of valid JSON as defined by the <a href=http://json.org/>JSON specification</a></p><p>The sort order used for <code>JSON</code> is unsigned byte-wise comparison.</p><h3 id=bson>BSON</h3><p><code>BSON</code> is used for an embedded BSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as an encoded BSON document as
+defined by the <a href=http://bsonspec.org/spec.html>BSON specification</a>.</p><p>The sort order used for <code>BSON</code> is unsigned byte-wise comparison.</p><h3 id=variant>VARIANT</h3><p><code>VARIANT</code> is used for a Variant value. It must annotate a group. The group must
+contain a field named <code>metadata</code> and a field named <code>value</code>. Both fields must have
+type <code>binary</code>, which is also called <code>BYTE_ARRAY</code> in the Parquet thrift definition.
+The <code>VARIANT</code> annotated group can be used to store either an unshredded Variant
+value, or a shredded Variant value.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</code> must be of type <code>binary</code> (called <code>BYTE_ARRAY</code>
+in the Parquet thrift definition).</li><li>The <code>metadata</code> field is required and must be a valid Variant metadata component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>When present, the <code>value</code> field must be a valid Variant value component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>The <code>value</code> field is required for unshredded Variant values.</li><li>The <code>value</code> field is optional and may be null only when parts of the Variant
+value are shredded according to the <a href=/docs/file-format/types/variantshredding/>Variant shredding specification</a>.</li></ul><p>This is the expected representation of an unshredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_unshredded (VARIANT) {
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_shredded (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><h3 id=geometry>GEOMETRY</h3><p><code>GEOMETRY</code> is used for geospatial features in the Well-Known Binary (WKB) format
+with linear/planar edges interpolation. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has only one type parameter:</p><ul><li><code>crs</code>: An optional string value for CRS. If unset, the CRS defaults to
+<code>"OGC:CRS84"</code>, which means that the geometries must be stored in longitude,
+latitude based on the WGS84 datum.</li></ul><p>The sort order used for <code>GEOMETRY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h3 id=geography>GEOGRAPHY</h3><p><code>GEOGRAPHY</code> is used for geospatial features in the WKB format with an explicit
+(non-linear/non-planar) edges interpolation algorithm. It must annotate a
+<code>BYTE_ARRAY</code> primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has two type parameters:</p><ul><li><code>crs</code>: An optional string value for CRS. It must be a geographic CRS, where
+longitudes are bound by [-180, 180] and latitudes are bound by [-90, 90].
+If unset, the CRS defaults to <code>"OGC:CRS84"</code>.</li><li><code>algorithm</code>: An optional enum value to describes the edge interpolation
+algorithm. Supported values are: <code>SPHERICAL</code>, <code>VINCENTY</code>, <code>THOMAS</code>, <code>ANDOYER</code>,
+<code>KARNEY</code>. If unset, the algorithm defaults to <code>SPHERICAL</code>.</li></ul><p>The sort order used for <code>GEOGRAPHY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=nested-types>Nested Types</h2><p>This section specifies how <code>LIST</code> and <code>MAP</code> can be used to encode nested types
+by adding group levels around repeated fields that are not present in the data.</p><p>This does not affect repeated fields that are not annotated: A repeated field
+that is neither contained by a <code>LIST</code>- or <code>MAP</code>-annotated group nor annotated
+by <code>LIST</code> or <code>MAP</code> should be interpreted as a required list of required
+elements where the element type is the type of the field.</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// List<Integer> (non-null list, non-null elements)
+repeated int32 num;
+
+// List<Tuple<Integer, String>> (non-null list, non-null elements)
+repeated group my_list {
+ required int32 num;
+ optional binary str (STRING);
+}
+</code></pre><p>For all fields in the schema, implementations should use either <code>LIST</code> and
+<code>MAP</code> annotations <em>or</em> unannotated repeated fields, but not both. When using
+the annotations, no unannotated repeated types are allowed.</p><h3 id=lists>Lists</h3><p><code>LIST</code> is used to annotate types that should be interpreted as lists.</p><p><code>LIST</code> must always annotate a 3-level structure:</p><pre tabindex=0><code><list-repetition> group <name> (LIST) {
+ repeated group list {
+ <element-repetition> <element-type> element;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>LIST</code> that contains a
+single field named <code>list</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the list is nullable.</li><li>The middle level, named <code>list</code>, must be a repeated group with a single
+field named <code>element</code>.</li><li>The <code>element</code> field encodes the list’s element type and repetition. Element
+repetition must be <code>required</code> or <code>optional</code>.</li></ul><p>The following examples demonstrate two of the possible lists of string values.</p><pre tabindex=0><code>// List<String> (list non-null, elements nullable)
+required group my_list (LIST) {
+ repeated group list {
+ optional binary element (STRING);
+ }
+}
+
+// List<String> (list nullable, elements non-null)
+optional group my_list (LIST) {
+ repeated group list {
+ required binary element (STRING);
+ }
+}
+</code></pre><p>Element types can be nested structures. For example, a list of lists:</p><pre tabindex=0><code>// List<List<Integer>>
+optional group array_of_arrays (LIST) {
+ repeated group list {
+ required group element (LIST) {
+ repeated group list {
+ required int32 element;
+ }
+ }
+ }
+}
+</code></pre><h4 id=backward-compatibility-rules>Backward-compatibility rules</h4><p>New writer implementations should always produce the 3-level LIST structure shown
+above. However, historically data files have been produced that use different
+structures to represent list-like data, and readers may include compatibility
+measures to interpret them as intended.</p><p>It is required that the repeated group of elements is named <code>list</code> and that
+its element field is named <code>element</code>. However, these names may not be used in
+existing data and should not be enforced as errors when reading. For example,
+the following field schema should produce a nullable list of non-null strings,
+even though the repeated group is named <code>element</code>.</p><pre tabindex=0><code>optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ };
+}
+</code></pre><p>Some existing data does not include the inner element layer, resulting in a
+<code>LIST</code> that annotates a 2-level structure. Unlike the 3-level structure, the
+repetition of a 2-level structure can be <code>optional</code>, <code>required</code>, or <code>repeated</code>.
+When it is <code>repeated</code>, the <code>LIST</code>-annotated 2-level structure can only serve as
+an element within another <code>LIST</code>-annotated 2-level structure.</p><p>For backward-compatibility, the type of elements in <code>LIST</code>-annotated structures
+should always be determined by the following rules:</p><ol><li>If the repeated field is not a group, then its type is the element type and
+elements are required.</li><li>If the repeated field is a group with multiple fields, then its type is the
+element type and elements are required.</li><li>If the repeated field is a group with one field with <code>repeated</code> repetition,
+then its type is the element type and elements are required.</li><li>If the repeated field is a group with one field and is named either <code>array</code>
+or uses the <code>LIST</code>-annotated group’s name with <code>_tuple</code> appended then the
+repeated type is the element type and elements are required.</li><li>Otherwise, the repeated field’s type is the element type with the repeated
+field’s repetition.</li></ol><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// Rule 1: List<Integer> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated int32 element;
+}
+
+// Rule 2: List<Tuple<String, Integer>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ required int32 num;
+ };
+}
+
+// Rule 3: List<List<Integer>> (nullable outer list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array (LIST) {
+ repeated int32 array;
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array {
+ required binary str (STRING);
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group my_list_tuple {
+ required binary str (STRING);
+ };
+}
+
+// Rule 5: List<String> (nullable list, nullable elements)
+optional group my_list (LIST) {
+ repeated group element {
+ optional binary str (STRING);
+ };
+}
+</code></pre><h3 id=maps>Maps</h3><p><code>MAP</code> is used to annotate types that should be interpreted as a map from keys
+to values. <code>MAP</code> must annotate a 3-level structure:</p><pre tabindex=0><code><map-repetition> group <name> (MAP) {
+ repeated group key_value {
+ required <key-type> key;
+ <value-repetition> <value-type> value;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>MAP</code> that contains a
+single field named <code>key_value</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the map is nullable.</li><li>The middle level, named <code>key_value</code>, must be a repeated group with a <code>key</code>
+field for map keys and, optionally, a <code>value</code> field for map values. It must
+not contain any other values.</li><li>The <code>key</code> field encodes the map’s key type. This field must have
+repetition <code>required</code> and must always be present. It must always be the first
+field of the repeated <code>key_value</code> group.</li><li>The <code>value</code> field encodes the map’s value type and repetition. This field can
+be <code>required</code>, <code>optional</code>, or omitted. It must always be the second field of
+the repeated <code>key_value</code> group if present. In case of not present, it can be
+represented as a map with all null values or as a set of keys.</li></ul><p>The following example demonstrates the type for a non-null map from strings to
+nullable integers:</p><pre tabindex=0><code>// Map<String, Integer>
+required group my_map (MAP) {
+ repeated group key_value {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><p>If there are multiple key-value pairs for the same key, then the final value
+for that key must be the last value. Other values may be ignored or may be
+added with replacement to the map container in the order that they are encoded.
+The <code>MAP</code> annotation should not be used to encode multi-maps using duplicate
+keys.</p><h4 id=backward-compatibility-rules-1>Backward-compatibility rules</h4><p>It is required that the repeated group of key-value pairs is named <code>key_value</code>
+and that its fields are named <code>key</code> and <code>value</code>. However, these names may not
+be used in existing data and should not be enforced as errors when reading.
+(<code>key</code> and <code>value</code> can be identified by their position in case of misnaming.)</p><p>Some existing data incorrectly used <code>MAP_KEY_VALUE</code> in place of <code>MAP</code>. For
+backward-compatibility, a group annotated with <code>MAP_KEY_VALUE</code> that is not
+contained by a <code>MAP</code>-annotated group should be handled as a <code>MAP</code>-annotated
+group.</p><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>// Map<String, Integer> (nullable map, non-null values)
+optional group my_map (MAP) {
+ repeated group map {
+ required binary str (STRING);
+ required int32 num;
+ }
+}
+
+// Map<String, Integer> (nullable map, nullable values)
+optional group my_map (MAP_KEY_VALUE) {
+ repeated group map {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><h2 id=unknown-always-null>UNKNOWN (always null)</h2><p>Sometimes, when discovering the schema of existing data, values are always null
+and there’s no type information.
+The <code>UNKNOWN</code> type can be used to annotate a column that is always null.
+(Similar to Null type in Avro and Arrow)</p></div><div class=td-content style=page-break-before:always><h1 id=pg-29a57736b5ceb62cc6d5a0ba8ba429b0>3.5.3 -</h1><h1 id=variant-shredding>Variant Shredding</h1><p>The Variant type is designed to store and process semi-structured data efficiently, even with heterogeneous values.
+Query engines encode each Variant value in a self-describing format, and store it as a group containing <code>value</code> and <code>metadata</code> binary fields in Parquet.
+Since data is often partially homogeneous, it can be beneficial to extract certain fields into separate Parquet columns to further improve performance.
+This process is called <strong>shredding</strong>.</p><p>Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections.</p><p>For example, the query <code>SELECT variant_get(event, '$.event_ts', 'timestamp') FROM tbl</code> only needs to load field <code>event_ts</code>, and if that column is shredded, it can be read by columnar projection without reading or deserializing the rest of the < [...]
+Similarly, for the query <code>SELECT * FROM tbl WHERE variant_get(event, '$.event_type', 'string') = 'signup'</code>, the <code>event_type</code> shredded column metadata can be used for skipping and to lazily load the rest of the Variant.</p><h2 id=variant-metadata>Variant Metadata</h2><p>Variant metadata is stored in the top-level Variant group in a binary <code>metadata</code> column regardless of whether the Variant value is shredded.</p><p>All <code>value</code> columns within the [...]
+All field names of a Variant, whether shredded or not, must be present in the metadata.</p><h2 id=value-shredding>Value Shredding</h2><p>Variant values are stored in Parquet fields named <code>value</code>.
+Each <code>value</code> field may have an associated shredded field named <code>typed_value</code> that stores the value when it matches a specific type.
+When <code>typed_value</code> is present, readers <strong>must</strong> reconstruct shredded values according to this specification.</p><p>For example, a Variant field, <code>measurement</code> may be shredded as long values by adding <code>typed_value</code> with type <code>int64</code>:</p><pre tabindex=0><code>required group measurement (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The Parquet columns used to store variant metadata and values must be accessed by name, not by position.</p><p>The series of measurements <code>34, null, "n/a", 100</code> would be stored as:</p><table><thead><tr><th>Value</th><th><code>metadata</code></th><th><code>value</code></th><th><code>typed_value</code></th></tr></thead><tbody><tr><td>34</td><td><code>01 00</code> v1/empty</td><td>null</td><td><code>34</code></td></tr><tr><td>null</td><td><code>01 00</code> v1/emp [...]
+Values in the two fields must be interpreted according to the following table:</p><table><thead><tr><th><code>value</code></th><th><code>typed_value</code></th><th>Meaning</th></tr></thead><tbody><tr><td>null</td><td>null</td><td>The value is missing; only valid for shredded object fields</td></tr><tr><td>non-null</td><td>null</td><td>The value is present and may be any type, including null</td></tr><tr><td>null</td><td>non-null</td><td>The value is present and is the shredded type</td>< [...]
+Writers must not produce data where both <code>value</code> and <code>typed_value</code> are non-null, unless the Variant value is an object.</p><p>If a Variant is missing in a context where a value is required, readers must return a Variant null (<code>00</code>): basic type 0 (primitive) and physical type 0 (null).
+For example, if a Variant is required (like <code>measurement</code> above) and both <code>value</code> and <code>typed_value</code> are null, the returned <code>value</code> must be <code>00</code> (Variant null).</p><h3 id=shredded-value-types>Shredded Value Types</h3><p>Shredded values must use the following Parquet types:</p><table><thead><tr><th>Variant Type</th><th>Parquet Physical Type</th><th>Parquet Logical Type</th></tr></thead><tbody><tr><td>boolean</td><td>BOOLEAN</td><td></t [...]
+If the value is an array, <code>value</code> must be null.</p><p>The list <code>element</code> must be a required group.
+The <code>element</code> group can contain <code>value</code> and <code>typed_value</code> fields.
+The element’s <code>value</code> field stores the element as Variant-encoded <code>binary</code> when the <code>typed_value</code> is not present or cannot represent it.
+The <code>typed_value</code> field may be omitted when not shredding elements as a specific type.
+The <code>value</code> field may be omitted when shredding elements as a specific type.
+However, at least one of the two fields must be present.</p><p>For example, a <code>tags</code> Variant may be shredded as a list of strings using the following definition:</p><pre tabindex=0><code>optional group tags (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value (LIST) { # must be optional to allow a null list
+ repeated group list {
+ required group element { # shredded element
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+}
+</code></pre><p>All elements of an array must be present (not missing) because the <code>array</code> Variant encoding does not allow missing elements.
+That is, either <code>typed_value</code> or <code>value</code> (but not both) must be non-null.
+Null elements must be encoded in <code>value</code> as Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>The series of <code>tags</code> arrays <code>["comedy", "drama"], ["horror", null], ["comedy", "drama", "romance"], null</code> would be stored as:</p><table><thead><tr><th>Array</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value...value</code></th><th><code>typed_value...typed_value</code></th></tr></thead><tbody><tr><td><code [...]
+If the value is not an object, <code>typed_value</code> must be null.
+Readers can assume that a value is not an object if <code>typed_value</code> is null and that <code>typed_value</code> field values are correct; that is, readers do not need to read the <code>value</code> column if <code>typed_value</code> fields satisfy the required fields.</p><p>Each shredded field in the <code>typed_value</code> group is represented as a required group that contains optional <code>value</code> and <code>typed_value</code> fields.
+The <code>value</code> field stores the value as Variant-encoded <code>binary</code> when the <code>typed_value</code> cannot represent the field.
+This layout enables readers to skip data based on the field statistics for <code>value</code> and <code>typed_value</code>.
+The <code>typed_value</code> field may be omitted when not shredding fields as a specific type.</p><p>The <code>value</code> column of a partially shredded object must never contain fields represented by the Parquet columns in <code>typed_value</code> (shredded fields).
+Readers may always assume that data is written correctly and that shredded fields in <code>typed_value</code> are not present in <code>value</code>.
+As a result, reads when a field is defined in both <code>value</code> and a <code>typed_value</code> shredded field may be inconsistent.</p><p>For example, a Variant <code>event</code> field may shred <code>event_type</code> (<code>string</code>) and <code>event_ts</code> (<code>timestamp</code>) columns using the following definition:</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value; # a variant, expected to be an object
+ optional group typed_value { # shredded fields for the variant object
+ required group event_type { # shredded field for event_type
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts { # shredded field for event_ts
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ }
+}
+</code></pre><p>The group for each named field must use repetition level <code>required</code>.</p><p>A field’s <code>value</code> and <code>typed_value</code> are set to null (missing) to indicate that the field does not exist in the variant.
+To encode a field that is present with a null value, the <code>value</code> must contain a Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>When both <code>value</code> and <code>typed_value</code> for a field are non-null, engines should fail.
+If engines choose to read in such cases, then the <code>typed_value</code> column must be used.
+Readers may always assume that data is written correctly and that only <code>value</code> or <code>typed_value</code> is defined.
+As a result, reads when both <code>value</code> and <code>typed_value</code> are defined may be inconsistent with optimized reads that require only one of the columns.</p><p>The table below shows how the series of objects in the first column would be stored:</p><table><thead><tr><th>Event object</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value.event_type.value</code></th><th><code>typed_value.event_type.typed_value</code></th><th><code>typed_value.eve [...]
+Readers must return an object when <code>typed_value</code> is non-null containing the shredded fields.</p><h2 id=nesting>Nesting</h2><p>The <code>typed_value</code> associated with any Variant <code>value</code> field can be any shredded type, as shown in the sections above.</p><p>For example, the <code>event</code> object above may also shred sub-fields as object (<code>location</code>) or array (<code>tags</code>).</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value {
+ required group event_type {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts {
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ required group location {
+ optional binary value;
+ optional group typed_value {
+ required group latitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ required group longitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ }
+ }
+ required group tags {
+ optional binary value;
+ optional group typed_value (LIST) {
+ repeated group list {
+ required group element {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+ }
+ }
+}
+</code></pre><h1 id=data-skipping>Data Skipping</h1><p>Statistics for <code>typed_value</code> columns can be used for file, row group, or page skipping when <code>value</code> is always null (missing).</p><p>When the corresponding <code>value</code> column is all nulls, all values must be the shredded <code>typed_value</code> field’s type.
+Because the type is known, comparisons with values of that type are valid.
+<code>IS NULL</code>/<code>IS NOT NULL</code> and <code>IS NAN</code>/<code>IS NOT NAN</code> filter results are also valid.</p><p>Comparisons with values of other types are not necessarily valid and data should not be skipped.</p><p>Casting behavior for Variant is delegated to processing engines.
+For example, the interpretation of a string as a timestamp may depend on the engine’s SQL session time zone.</p><h2 id=reconstructing-a-shredded-variant>Reconstructing a Shredded Variant</h2><p>It is possible to recover an unshredded Variant value using a recursive algorithm, where the initial call is to <code>construct_variant</code> with the top-level Variant group fields.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-siz [...]
+</span></span><span style=display:flex><span> <span style=color:#4e9a06>"""Constructs a Variant from value and typed_value"""</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>typed_value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>dict</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#000>object_fields</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#000>name</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> < [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>name</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#000>typed_value</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a partially shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>),</span> <span style=color:#4e9a06>"partially shredded value must be an object"</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>typed_value</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>keys</span><span style=color:#000;font-weight:700>()</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>isdisjoint</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#ce5c00;font [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># union the shredded fields and non-shredded fields</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>union</span><span style=color:#000;f [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded array</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded array must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#000>elements</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>[</span>
+</span></span><span style=display:flex><span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span> [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000>elem</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantArray</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elements</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded primitive</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded primitive must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># value is missing</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#204a87;font-weight:700>None</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>def</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>Any</span><span style=color:#000;font-weight:700>):</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>int</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantInteger</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>str</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantString</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>...</span>
+</span></span></code></pre></div><h2 id=backward-and-forward-compatibility>Backward and forward compatibility</h2><p>Shredding is an optional feature of Variant, and readers must continue to be able to read a group containing only <code>value</code> and <code>metadata</code> fields.</p><p>Engines that do not write shredded values must be able to read shredded values according to this spec or must fail.</p><p>Different files may contain conflicting shredding schemas.
+That is, files may contain different <code>typed_value</code> columns for the same Variant with incompatible types.
+It may not be possible to infer or specify a single shredded schema that would allow all Parquet files for a table to be read without reconstructing the value as a Variant.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-ad7f0d170ee709f2e58f9e27d7567421>3.5.4 -</h1><h1 id=variant-binary-encoding>Variant Binary Encoding</h1><p>A Variant represents a type that contains one of:</p><ul><li>Primitive: A type and corresponding value (e.g. INT, STRING)</li><li>Array: An [...]
+These represent a commonly supported subset of the <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>logical types</a> allowed by the Parquet format.</p><p>The Variant Binary Encoding allows representation of semi-structured data (e.g. JSON) in a form that can be efficiently queried by path.
+The design is intended to allow efficient access to nested data even in the presence of very wide or deep structures.</p><p>Another motivation for the representation is that (aside from metadata) each nested Variant value is contiguous and self-contained.
+For example, in a Variant containing an Array of Variant values, the representation of an inner Variant value, when paired with the metadata of the full variant, is itself a valid Variant.</p><p>This document describes the Variant Binary Encoding scheme.
+Variant fields can also be <em>shredded</em>.
+Shredding refers to extracting some elements of the variant into separate columns for more efficient extraction/filter pushdown.
+The <a href=/docs/file-format/types/variantshredding/>Variant Shredding specification</a> describes the details of shredding Variant values as typed Parquet columns.</p><h2 id=variant-in-parquet>Variant in Parquet</h2><p>A Variant value in Parquet is represented by a group with 2 fields, named <code>value</code> and <code>metadata</code>.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</c [...]
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group shredded_variant_name (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The <code>VARIANT</code> annotation places no additional restrictions on the repetition of Variant groups, but repetition may be restricted by containing types (such as <code>MAP</code> and <code>LIST</code>).
+The Variant group name is the name of the Variant column.</p><h2 id=metadata-encoding>Metadata encoding</h2><p>The encoded metadata always starts with a header byte.</p><pre tabindex=0><code> 7 6 5 4 3 0
+ +-------+---+---+---------------+
+header | | | | version |
+ +-------+---+---+---------------+
+ ^ ^
+ | +-- sorted_strings
+ +-- offset_size_minus_one
+</code></pre><p>The <code>version</code> is a 4-bit value that must always contain the value <code>1</code>.
+<code>sorted_strings</code> is a 1-bit value indicating whether dictionary strings are sorted and unique.
+<code>offset_size_minus_one</code> is a 2-bit value providing the number of bytes per dictionary size and offset field.
+The actual number of bytes, <code>offset_size</code>, is <code>offset_size_minus_one + 1</code>.</p><p>The entire metadata is encoded as the following diagram shows:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+metadata | header |
+ +-----------------------+
+ | |
+ : dictionary_size : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | | (`dictionary_size + 1` offsets)
+ +-----------------------+
+ | |
+ : bytes :
+ | |
+ +-----------------------+
+</code></pre><p>The metadata is encoded first with the <code>header</code> byte, then <code>dictionary_size</code> which is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the number of string values in the dictionary.
+Next, is an <code>offset</code> list, which contains <code>dictionary_size + 1</code> values.
+Each <code>offset</code> is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the starting byte offset of the i-th string in <code>bytes</code>.
+The first <code>offset</code> value will always be <code>0</code>, and the last <code>offset</code> value will always be the total length of <code>bytes</code>.
+The last part of the metadata is <code>bytes</code>, which stores all the string values in the dictionary.
+All string values must be UTF-8 encoded strings.</p><h3 id=metadata-encoding-grammar>Metadata encoding grammar</h3><p>The grammar for encoded metadata is as follows</p><pre tabindex=0><code>metadata: <header> <dictionary_size> <dictionary>
+header: 1 byte (<version> | <sorted_strings> << 4 | (<offset_size_minus_one> << 6))
+version: a 4-bit version ID. Currently, must always contain the value 1
+sorted_strings: a 1-bit value indicating whether metadata strings are sorted
+offset_size_minus_one: 2-bit value providing the number of bytes per dictionary size and offset field.
+dictionary_size: `offset_size` bytes. unsigned little-endian value indicating the number of strings in the dictionary
+dictionary: <offset>* <bytes>
+offset: `offset_size` bytes. unsigned little-endian value indicating the starting position of the ith string in `bytes`. The list should contain `dictionary_size + 1` values, where the last value is the total length of `bytes`.
+bytes: UTF-8 encoded dictionary string values
+</code></pre><p>Notes:</p><ul><li>Offsets are relative to the start of the <code>bytes</code> array.</li><li>The length of the ith string can be computed as <code>offset[i+1] - offset[i]</code>.</li><li>The offset of the first string is always equal to 0 and is therefore redundant. It is included in the spec to simplify in-memory-processing.</li><li><code>offset_size_minus_one</code> indicates the number of bytes per <code>dictionary_size</code> and <code>offset</code> entry. I.e. a valu [...]
+ +------------------------------------+------------+
+value | value_header | basic_type | <-- value_metadata
+ +------------------------------------+------------+
+ | |
+ : value_data : <-- 0 or more bytes
+ | |
+ +-------------------------------------------------+
+</code></pre><h3 id=basic-type>Basic Type</h3><p>The <code>basic_type</code> is 2-bit value that represents which basic type the Variant value is.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>basic types table</a> shows what each value represents.</p><h3 id=value-header>Value Header</h3><p>The <code>value_header</code> is a 6-bit value that contains more information about the type, and the format depends on the <code>basic_type</code>.</p><h4 id=value-header-for-primitive-type-basic_type0>Value Header for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>v [...]
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows what each value represents.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | primitive_header |
+ +-----------------------+
+</code></pre><h4 id=value-header-for-short-string-basic_type1>Value Header for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_header</code> is a 6-bit <code>short_string_header</code>.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | short_string_header |
+ +-----------------------+
+</code></pre><p>The <code>short_string_header</code> value is the length of the string.</p><h4 id=value-header-for-object-basic_type2>Value Header for Object (<code>basic_type</code>=2)</h4><p>When <code>basic_type</code> is <code>2</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, <code>field_id_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 4 3 2 1 0
+ +---+---+-------+-------+
+value_header | | | | |
+ +---+---+-------+-------+
+ ^ ^ ^
+ | | +-- field_offset_size_minus_one
+ | +-- field_id_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> and <code>field_id_size_minus_one</code> are 2-bit values that represent the number of bytes used to encode the field offsets and field ids.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code> and <code>field_id_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h4 id=value-header-for-array-basic_type3>Value Header for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 3 2 1 0
+ +-----------+---+-------+
+value_header | | | |
+ +-----------+---+-------+
+ ^ ^
+ | +-- field_offset_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> is a 2-bit value that represents the number of bytes used to encode the field offset.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h3 id=value-data>Value Data</h3><p>The <code>value_data</code> encoding format depends on the type specified by <code>value_metadata</code>.
+For some types, the <code>value_data</code> will be 0-bytes.</p><h4 id=value-data-for-primitive-type-basic_type0>Value Data for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>value_data</code> depends on the <code>primitive_header</code> value.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows the encoding format for each primitive type.</p><h4 id=value-data-for-short-string-basic_type1>Value Data for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_data</code> is the sequence of UTF-8 encoded bytes that represents the string.</p><h4 id=value-data-for-object-basic_type2>Value Data for Object (<code>basic_type</code>=2)</ [...]
+The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+object value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | | (`num_elements` field_ids)
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An object <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the object.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a list of <code>field_id</code> values.
+There are <code>num_elements</code> number of entries and each <code>field_id</code> is an unsigned little-endian value of <code>field_id_size</code> bytes.
+A <code>field_id</code> is an index into the dictionary in the metadata.
+The <code>field_id</code> list is followed by a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the end of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th key-value pair of the object, the key is the metadata dictionary entry indexed by the i-th <code>field_id</code>, and the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><p>The field ids and field offsets must be in lexicographical order of the corresponding field names in the metadata dictionary.
+However, the actual <code>value</code> entries do not need to be in any particular order.
+This implies that the <code>field_offset</code> values may not be monotonically increasing.
+For example, for the following object:</p><pre tabindex=0><code>{
+ "c": 3,
+ "b": 2,
+ "a": 1
+}
+</code></pre><p>The <code>field_id</code> list must be <code>[<id for key "a">, <id for key "b">, <id for key "c">]</code>, in lexicographical order.
+The <code>field_offset</code> list must be <code>[<offset for value 1>, <offset for value 2>, <offset for value 3>, <last offset>]</code>.
+The <code>value</code> list can be in any order.</p><h4 id=value-data-for-array-basic_type3>Value Data for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_data</code> encodes an array. The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+array value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An array <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the array.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the last byte of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th array entry, the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><h3 id=value-encoding-grammar>Value encoding grammar</h3><p>The grammar for an encoded value is:</p><pre tabindex=0><code>value: <value_metadata> <value_data>?
+value_metadata: 1 byte (<basic_type> | (<value_header> << 2))
+basic_type: ID from Basic Type table. <value_header> must be a corresponding variation
+value_header: <primitive_header> | <short_string_header> | <object_header> | <array_header>
+primitive_header: ID from Primitive Type table. <val> must be a corresponding variation of <primitive_val>
+short_string_header: unsigned string length in bytes from 0 to 63
+object_header: (is_large << 4 | field_id_size_minus_one << 2 | field_offset_size_minus_one)
+array_header: (is_large << 2 | field_offset_size_minus_one)
+value_data: <primitive_val> | <short_string_val> | <object_val> | <array_val>
+primitive_val: see table for binary representation
+short_string_val: UTF-8 encoded bytes
+object_val: <num_elements> <field_id>* <field_offset>* <fields>
+array_val: <num_elements> <field_offset>* <fields>
+num_elements: a 1 or 4 byte unsigned little-endian value (depending on is_large in <object_header>/<array_header>)
+field_id: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_id_size_minus_one in <object_header>), indexing into the dictionary
+field_offset: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_offset_size_minus_one in <object_header>/<array_header>), providing the offset in bytes within fields
+fields: <value>*
+</code></pre><p>Each <code>value_data</code> must correspond to the type defined by <code>value_metadata</code>. Boolean and null types do not have a corresponding <code>value_data</code>, since their type defines their value.</p><p>Each <code>array_val</code> and <code>object_val</code> must contain exactly <code>num_elements + 1</code> values for <code>field_offset</code>.
+The last entry is the offset that is one byte past the last field (i.e. the total size of all fields in bytes).
+All offsets are relative to the first byte of the first field in the object/array.</p><p><code>field_id_size_minus_one</code> and <code>field_offset_size_minus_one</code> indicate the number of bytes per field ID/offset.
+For example, a value of 0 indicates 1-byte IDs, 1 indicates 2-byte IDs, 2 indicates 3 byte IDs and 3 indicates 4-byte IDs.
+The <code>is_large</code> flag for arrays and objects is used to indicate whether the number of elements is indicated using a one or four byte value.
+When more than 255 elements are present, <code>is_large</code> must be set to true.
+It is valid for an implementation to use a larger value than necessary for any of these fields (e.g. <code>is_large</code> may be true for an object with less than 256 elements).</p><p>The “short string” basic type may be used as an optimization to fold string length into the type byte for strings less than 64 bytes.
+It is semantically identical to the “string” primitive type.</p><p>The Decimal type contains a scale, but no precision. The implied precision of a decimal value is <code>floor(log_10(val)) + 1</code>.</p><h2 id=encoding-types>Encoding types</h2><p><em>Variant basic types</em></p><table><thead><tr><th>Basic Type</th><th>ID</th><th>Description</th></tr></thead><tbody><tr><td>Primitive</td><td><code>0</code></td><td>One of the primitive types</td></tr><tr><td>Short string</td><t [...]
+For example, a user expression operating on a string value containing “hello” should behave the same, whether it is encoded with the short string optimization, or long string encoding.
+Similarly, user expressions operating on an <em>int8</em> value of 1 should behave the same as a decimal16 with scale 2 and unscaled value 100.</p><p><em>Decimal table</em></p><table><thead><tr><th>Decimal Precision</th><th>Decimal value type</th><th>Variant Physical Type</th></tr></thead><tbody><tr><td>1 <= precision <= 9</td><td>int32</td><td>decimal4</td></tr><tr><td>10 <= precision <= 18</td><td>int64</td><td>decimal8</td></tr><tr><td>19 <= precision <= 38</td><td>i [...]
+This includes the dictionary key string values, the “short string” values, and the “long string” values.</p><h2 id=object-field-id-order-and-uniqueness>Object field ID order and uniqueness</h2><p>For objects, field IDs and offsets must be listed in the order of the corresponding field names, sorted lexicographically (using unsigned byte ordering for UTF-8).
+Note that the field values themselves are not required to follow this order.
+As a result, offsets will not necessarily be listed in ascending order.
+The field values are not required to be in the same order as the field IDs, to enable flexibility when constructing Variant values.</p><p>An implementation may rely on this field ID order in searching for field names.
+E.g. a binary search on field IDs (combined with metadata lookups) may be used to find a field with a given name.</p><p>Field names are case-sensitive.
+Field names are required to be unique for each object.
+It is an error for an object to contain two fields with the same name, whether or not they have distinct dictionary IDs.</p><h2 id=versions-and-extensions>Versions and extensions</h2><p>An implementation is not expected to parse a Variant value whose metadata version is higher than the version supported by the implementation.
+However, new types may be added to the specification without incrementing the version ID.
+In such a situation, an implementation should be able to read the rest of the Variant value if desired.</p><h2 id=shredding>Shredding</h2><p>A single Variant object may have poor read performance when only a small subset of fields are needed.
+A better approach is to create separate columns for individual fields, referred to as shredding or subcolumnarization.
+<a href=/docs/file-format/types/variantshredding/>VariantShredding.md</a> describes the Variant shredding specification in Parquet.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-0d8efcf7d4835e2ce724ae57b02d4318>3.6 - Nested Encoding</h1><p>To encode nested columns, Parquet uses the Dremel encoding with definition and
repetition levels. Definition levels specify how many optional fields in the
path for the column are defined. Repetition levels specify at what repeated field
in the path has the value repeated. The max definition and repetition levels can
be computed from the schema (i.e. how much nesting there is). This defines the
maximum number of bits required to store the levels (levels are defined for all
-values in the column).</p><p>Two encodings for the levels are supported BIT_PACKED and RLE. Only RLE is now used as it supersedes BIT_PACKED.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cf811a6e949c01acbf67ad2370adc5c8>3.6 - Bloom Filter</h1><h3 id=problem-statement>Problem statement</h3><p>In their current format, column statistics and dictionaries can be used for predicate
+values in the column).</p><p>Two encodings for the levels are supported BIT_PACKED and RLE. Only RLE is now used as it supersedes BIT_PACKED.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cf811a6e949c01acbf67ad2370adc5c8>3.7 -</h1><h1 id=parquet-bloom-filter>Parquet Bloom Filter</h1><h3 id=problem-statement>Problem statement</h3><p>In their current format, column statistics and dictionaries can be used for predicate
pushdown. Statistics include minimum and maximum value, which can be used to filter out
values not in the range. Dictionaries are more specific, and readers can filter out values
that are between min and max but not in the dictionary. However, when there are too many
@@ -209,7 +1081,7 @@ significant 32 bits.</p><pre tabindex=0><code>void filter_insert(SBBF filter, un
block b = filter.getBlock(i);
return block_check(b, (unsigned int32)x)
}
-</code></pre><p>The use of blocks is from Putze et al.’s <a href=https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf>Cache-, Hash- and
+</code></pre><p>The use of blocks is from Putze et al.’s <a href=http://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf>Cache-, Hash- and
Space-Efficient Bloom
filters</a></p><p>To use an SBBF for values of arbitrary Parquet types, we apply a hash
function to that value - at the time of writing,
@@ -287,8 +1159,8 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src=/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
-<img src=/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
+<img src=/parquet-format/doc/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
+<img src=/parquet-format/doc/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
data should be encrypted with the column key, and the Bloom filter of other (not sensitive) columns
do not need to be encrypted.</p><p>Bloom filters have two serializable modules - the PageHeader thrift structure (with its internal
@@ -298,17 +1170,17 @@ serialized Bitset.</p><p>For Bloom filters in sensitive columns, each of the two
serialization, and then written to the file. The encryption will be performed using the AES GCM
cipher, with the same column key, but with different AAD module types - “BloomFilter Header” (8)
and “BloomFilter Bitset” (9). The length of the encrypted buffer is written before the buffer, as
-described in the Parquet encryption specification.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cfdfda4ee78f351464c0ddf31e278a88>3.7 - Data Pages</h1><p>For data pages, the 3 pieces of information are encoded back to back, after the page
+described in the Parquet encryption specification.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-cfdfda4ee78f351464c0ddf31e278a88>3.8 - Data Pages</h1><p>For data pages, the 3 pieces of information are encoded back to back, after the page
header. No padding is allowed in the data page.
In order we have:</p><ol><li>repetition levels data</li><li>definition levels data</li><li>encoded values</li></ol><p>The value of <code>uncompressed_page_size</code> specified in the header is for all the 3 pieces combined.</p><p>The encoded values for the data page is always required. The definition and repetition levels
are optional, based on the schema definition. If the column is not nested (i.e.
the path to the column has length 1), we do not encode the repetition levels (it would
always have the value 1). For data that is required, the definition levels are
skipped (if encoded, it will always have the value of the max definition level).</p><p>For example, in the case where the column is non-nested and required, the data in the
-page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p></div><div class=td-content style=page-break-before:always><h1 id=pg-1745a7620ee7ae9962c3e6baaa083e89>3.7.1 - Compression</h1><h2 id=overview>Overview</h2><p>Parquet allows the data block inside dictionary pages and data pages to
+page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p></div><div class=td-content style=page-break-before:always><h1 id=pg-1745a7620ee7ae9962c3e6baaa083e89>3.8.1 -</h1><h1 id=parquet-compression-definitions>Parquet compression definitions</h1><p>This document contains the specification of all supported compression codecs.</p><h2 id=overview>Overview</h2><p>Parquet allows the d [...]
be compressed for better space efficiency. The Parquet format supports
-several compression covering different areas in the compression ratio /
-processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
+several compression codecs covering different areas in the compression
+ratio / processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
by their respective authors or maintainers, which we reference hereafter.</p><p>For all compression codecs except the deprecated <code>LZ4</code> codec, the raw data
of a (data or dictionary) page is fed <em>as-is</em> to the underlying compression
library, without any additional framing or padding. The information required
@@ -338,8 +1210,8 @@ when implementing this format, the implementation provided by the
<a href=https://facebook.github.io/zstd/>Zstandard compression library</a>
is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md>LZ4 block format</a>.
If any ambiguity arises when implementing this format, the implementation
-provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>3.7.2 - Encodings</h1><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supported for types. It is
-intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding cannot be used. It
+provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-9aa971e751fdd370158d525ad337ef7a>3.8.2 -</h1><h1 id=parquet-encoding-definitions>Parquet encoding definitions</h1><p>This file contains the specification of all supported encodings.</p><p><a name=PLAIN></a></p><h3 id=plain-plain--0>Plain: (PLAIN = 0)</h3><p>Supported Types: all</p><p>This is the plain encoding that must be supp [...]
+intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
@@ -366,8 +1238,7 @@ repeated-value := value that is repeated, using a fixed-width of round-up-to-nex
</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
-significant. An example of the encoding is presented below. For comparison, the same case is shown
-in the example of the deprecated bit-packing encoding in the next section.</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 10001000 11000110 11111010
@@ -441,7 +1312,9 @@ by keeping track of the number of values read.</p><p>Subtractions in steps 1) an
will the corresponding additions when decoding. Overflow should be allowed
and handled as wrapping around in 2’s complement notation so that the original
values are correctly restituted. This may require explicit care in some programming
-languages (for example by doing all arithmetic in the unsigned domain).</p><p>The following examples use 8 as the block size to keep the examples short,
+languages (for example by doing all arithmetic in the unsigned domain). Writers
+must not use more bits when bit packing the miniblock data than would be required
+to PLAIN encode the physical type (e.g. INT32 data must not use more than 32 bits).</p><p>The following examples use 8 as the block size to keep the examples short,
but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2, 3, 4, 5</p><p>After step 1), we compute the deltas as:</p><p>1, 1, 1, 1</p><p>The minimum delta is 1 and after step 2, the relative deltas become:</p><p>0, 0, 0, 0</p><p>The final encoded data is:</p><p>header:
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
@@ -453,11 +1326,10 @@ concatenated back to back. The expected savings is from the cost of encoding the
and possibly better compression in the data (it is no longer interleaved with the lengths).</p><p>The data stream looks like:</p><pre tabindex=0><code><Delta Encoded Lengths> <Byte Array Data>
</code></pre><p>For example, if the data was “Hello”, “World”, “Foobar”, “ABCDEF”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(5, 5, 6, 6) (the string lengths)</li><li>“HelloWorldFoobarABCDEF”</li></ul><h3 id=delta-strings-delta_byte_array--7>Delta Strings: (DELTA_BYTE_ARRAY = 7)</h3><p>Supported Types: BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY</p><p>This is also known as incremental [...]
sequence of strings, store the prefix length of the previous entry plus the suffix.</p><p>For a longer description, see <a href=https://en.wikipedia.org/wiki/Incremental_encoding>https://en.wikipedia.org/wiki/Incremental_encoding</a>.</p><p>This is stored as a sequence of delta-encoded prefix lengths (DELTA_BINARY_PACKED), followed by
-the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
+the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
compression ratio and speed when a compression algorithm is used afterwards.</p><p>This encoding creates K byte-streams of length N where K is the size in bytes of the data
-type and N is the number of elements in the data sequence. Specifically, K is 4 for FLOAT
-type and 8 for DOUBLE type.
-The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
+type and N is the number of elements in the data sequence. For example, K is 4 for FLOAT
+type and 8 for DOUBLE type.</p><p>The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
0-th stream, the 1-st byte goes to the 1-st stream and so on.
The streams are concatenated in the following order: 0-th stream, 1-st stream, etc.
The total length of encoded streams is K * N bytes. Because it does not have any metadata
@@ -466,7 +1338,7 @@ is allowed inside the data page.</p><p>Example:
Original data is three 32-bit floats and for simplicity we look at their raw representation.</p><pre tabindex=0><code> Element 0 Element 1 Element 2
Bytes AA BB CC DD 00 11 22 33 A3 B4 C5 D6
</code></pre><p>After applying the transformation, the data has the following representation:</p><pre tabindex=0><code>Bytes AA 00 A3 BB 11 B4 CC 22 C5 DD 33 D6
-</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-70f861f07d9aae8f8d133f3351caad43>3.7.3 - Parquet Modular Encryption</h1><p>Parquet files containing sensitive information can be protected by the modular encryption
+</code></pre></div><div class=td-content style=page-break-before:always><h1 id=pg-70f861f07d9aae8f8d133f3351caad43>3.8.3 -</h1><h1 id=parquet-modular-encryption>Parquet Modular Encryption</h1><p>Parquet files containing sensitive information can be protected by the modular encryption
mechanism that encrypts and authenticates the file data and metadata - while allowing
for a regular Parquet functionality (columnar projection, predicate pushdown, encoding
and compression).</p><h2 id=1-problem-statement>1 Problem Statement</h2><p>Existing data protection solutions (such as flat encryption of files, in-storage encryption,
@@ -521,7 +1393,20 @@ The unique input to CTR encryption is called IV (“initialization vector&rd
parts: a nonce and an initial counter field.</p><p>Parquet encryption uses the RBG-based (random bit generator) nonce construction as defined in
the section 8.2.2 of the NIST SP 800-38D document. For each encrypted module, Parquet generates a
unique nonce with a length of 12 bytes (96 bits). Notice: the NIST
-specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
+specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h4 id=414-invocation-limit>4.1.4 Invocation limit</h4><p>According to the section 8.3 of the NIST SP 800-38D document, <em>“The total number of invocations
+of the authenticated encryption function shall not exceed 2^32, including all IV lengths and
+all instances of the authenticated encryption function with the given key”</em>. This restriction is
+related to the “uniqueness requirement of IVs and keys” (section 8 in the NIST spec) - <em>“if even
+one IV is ever repeated, then the implementation may be vulnerable”</em>. <em>“Compliance with this
+requirement is crucial to the security of GCM”</em>.</p><p>The bulk of modules in a Parquet file are page headers and data pages. Therefore, one encryption
+key shall not not be used for more than 2^31 (~2 billion) pages. In Parquet files encrypted with
+multiple keys (footer and column keys), the constraint on the number of invocations is applied
+to each key separately.</p><p>When running in the context of a larger system, any particular Parquet writer implementation likely
+does not have sufficient context to enforce key invocation limits system-wide. Therefore,
+the higher level system itself must arrange to supply keys appropriately to the various writer instances.</p><p>Parquet writer implementations should have a local invocation counter for each encryption key. If the
+counter exceeds 2^32, the implementation should return an error and produce no more cipherblocks.
+While this does not enforce a system-wide limit, it helps in distributed systems that provide different
+keys to different nodes (or generate unique keys in each node).</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
must be implemented by a cryptographic provider according to the NIST SP 800-38D specification.</p><p>In Parquet, an input to the GCM cipher is an encryption key, a 12-byte nonce, a plaintext and an
AAD. The output is a ciphertext with the length equal to that of plaintext, and a 16-byte authentication
tag used to verify the ciphertext and AAD integrity.</p><h4 id=422-aes_gcm_ctr_v1>4.2.2 AES_GCM_CTR_V1</h4><p>In this Parquet algorithm, all modules except pages are encrypted with the GCM cipher, as described
@@ -568,7 +1453,7 @@ data set (table). This string is optionally passed by a writer upon file creatio
the AAD prefix is stored in an <code>aad_prefix</code> field in the file, and is made available to the readers.
This field is not encrypted. If a user is concerned about keeping the file identity inside the file,
the writer code can explicitly request Parquet not to store the AAD prefix. Then the aad_prefix field
-will be empty; AAD prefixes must be fully managed by the caller code and supplied explicitly to Parquet
+will be empty; AAD prefixes must be fully managed by the caller code and supplied explictly to Parquet
readers for each file.</p><p>The protection against swapping full files is optional. It is not enabled by default because
it requires the writers to generate and pass an AAD prefix.</p><p>A reader of a file created with an AAD prefix, should be able to verify the prefix (file identity)
by comparing it with e.g. the target table name, using a convention accepted in the organization.
@@ -624,7 +1509,10 @@ decrypt it.</p><p>The row group ordinal, required for AAD suffix calculation, is
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/** Row group ordinal in the file **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>i16</span> <span style=color:#000>ordinal</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
+</span></span></code></pre></div><p>The integrity of this field is protected by authenticated encryption of the footer (FileMetaData). Therefore,
+the reader implementations can use either a local row group counter (ordinal) or the <code>RowGroup.ordinal</code>
+field as an input to AAD suffix calculation. The latter option can be helpful when different reader
+threads process different row groups in the same parquet file.</p><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
is a union - the actual structure is chosen depending on whether the column is encrypted with the
footer encryption key, or with a column-specific key. For the latter, a key metadata can be specified.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span> <span style=color:#000>EncryptionWithFooterKey</span> <span style=color:#000;font-weight:700>{</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
@@ -698,7 +1586,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img src=/parquet-format/doc/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -721,7 +1609,7 @@ structure with the
AES GCM algorithm - using a footer signing key, and an AAD constructed according to the instructions
of the section 4.4. Only the nonce and GCM tag are stored in the file – as a 28-byte
fixed-length array, written right after the footer itself. The ciphertext is not stored,
-because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</ [...]
+because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>stru [...]
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>...</span>
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/**
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * Encryption algorithm. This field is set only in encrypted files
@@ -742,17 +1630,17 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img src=/parquet-format/doc/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
to the default page size (1 MB). This is a rough estimation, and can change with the encryption
algorithm (no 16-byte tag in AES_GCM_CTR_V1) and with page configuration or data encoding/compression.</p><p>The throughput overhead of Parquet modular encryption depends on whether AES enciphering is
done in software or hardware. In both cases, performing encryption on full pages (~1MB buffers)
-instead of on much smaller individual data values causes AES to work at its maximal speed.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e4790b30bda91ce23679bbaa9c652cd0>3.7.4 - Checksumming</h1><p>Pages of all kinds can be individually checksummed. This allows disabling of checksums
+instead of on much smaller individual data values causes AES to work at its maximal speed.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e4790b30bda91ce23679bbaa9c652cd0>3.8.4 - Checksumming</h1><p>Pages of all kinds can be individually checksummed. This allows disabling of checksums
at the HDFS file level, to better support single row lookups. Checksums are calculated
using the standard CRC32 algorithm - as used in e.g. GZip - on the serialized binary
-representation of a page (not including the page header itself).</p></div><div class=td-content style=page-break-before:always><h1 id=pg-d84ba941c88c28699d637073c804ef79>3.7.5 - Column Chunks</h1><p>Column chunks are composed of pages written back to back. The pages share a common
+representation of a page (not including the page header itself).</p></div><div class=td-content style=page-break-before:always><h1 id=pg-d84ba941c88c28699d637073c804ef79>3.8.5 - Column Chunks</h1><p>Column chunks are composed of pages written back to back. The pages share a common
header and readers can skip over pages they are not interested in. The data for the
page follows the header and can be compressed and/or encoded. The compression and
encoding is specified in the page metadata.</p><p>A column chunk might be partly or completely dictionary encoded. It means that
@@ -761,7 +1649,7 @@ actual values are stored in the dictionary page. See details in Encodings.md.
The dictionary page must be placed at the first position of the column chunk. At
most one dictionary page can be placed in a column chunk.</p><p>Additionally, files can contain an optional column index to allow readers to
skip pages more efficiently. See PageIndex.md for details and
-the reasoning behind adding these to the format.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-3fa634598323d0f5132aedfc3988d28f>3.7.6 - Error Recovery</h1><p>If the file metadata is corrupt, the file is lost. If the column metadata is corrupt,
+the reasoning behind adding these to the format.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-3fa634598323d0f5132aedfc3988d28f>3.8.6 - Error Recovery</h1><p>If the file metadata is corrupt, the file is lost. If the column metadata is corrupt,
that column chunk is lost (but column chunks for this column in other row groups are
okay). If a page header is corrupt, the remaining pages in that chunk are lost. If
the data within a page is corrupt, that page is lost. The file will be more
@@ -771,12 +1659,14 @@ data written will be unreadable. This can be fixed by writing the file metadata
every Nth row group.
Each file metadata would be cumulative and include all the row groups written so
far. Combining this with the strategy used for rc or avro files using sync markers,
-a reader could recover partially written files.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-2c0e785302c1130bf2c45e6faa8aa18f>3.8 - Nulls</h1><p>Nullity is encoded in the definition levels (which is run-length encoded). NULL values
+a reader could recover partially written files.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-2c0e785302c1130bf2c45e6faa8aa18f>3.9 - Nulls</h1><p>Nullity is encoded in the definition levels (which is run-length encoded). NULL values
are not encoded in the data. For example, in a non-nested schema, a column with 1000 NULLs
would be encoded with run-length encoding (0, 1000 times) for the definition levels and
-nothing else.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-989662b6cba2c5cccd73a12b56828916>3.9 - Page Index</h1><p>This document describes the format for column index pages in the Parquet
-footer. These pages contain statistics for DataPages and can be used to skip
-pages when scanning data in ordered and unordered columns.</p><h2 id=problem-statement>Problem Statement</h2><p>In previous versions of the format, Statistics are stored for ColumnChunks in
+nothing else.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-989662b6cba2c5cccd73a12b56828916>3.10 -</h1><h1 id=parquet-page-index-layout-to-support-page-skipping>Parquet page index: Layout to Support Page Skipping</h1><p>In Parquet, a <em>page index</em> is optional metadata for a
+ColumnChunk, containing statistics for DataPages that can be used
+to skip those pages when scanning in ordered and unordered columns.
+The page index is stored using the OffsetIndex and ColumnIndex structures,
+defined in <a href=src/main/thrift/parquet.thrift><code>parquet.thrift</code></a></p><h2 id=problem-statement>Problem Statement</h2><p>In previous versions of the format, Statistics are stored for ColumnChunks in
ColumnMetaData and for individual pages inside DataPageHeader structs. When
reading pages, a reader had to process the page header to determine
whether the page could be skipped based on the statistics. This means the reader
@@ -798,7 +1688,7 @@ column are skipped, the corresponding rows in the other columns have to be
skipped. Hence the OffsetIndexes for each column in a RowGroup are stored
together.</li></ul><p>The new index structures are stored separately from RowGroup, near the footer.<br>This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
-location and length are stored in ColumnChunk.</p><p><img src=/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
+location and length are stored in ColumnChunk.</p><p><img src=/parquet-format/doc/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
bound for the last page, because the row group Statistics can provide that.
We still include those for the sake of uniformity, and the overhead should be
negligible.</li><li>We store lower and upper bounds for the values of each page. These may be the
@@ -816,7 +1706,7 @@ reader can find matching pages by sequentially reading <code>min_values</code> a
indices, and page offsets to scan in each column. The reader can then
initialize a scanner for each column and fast forward them to the start row of
the scan.</p><p>The <code>min_values</code> and <code>max_values</code> are calculated based on the <code>column_orders</code>
-field in the <code>FileMetaData</code> struct of the footer.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e0ad5830788d45de8b55e0c2b119349a>3.10 - Implementation status</h1><p>This page summarizes the features supported by different Parquet
+field in the <code>FileMetaData</code> struct of the footer.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-e0ad5830788d45de8b55e0c2b119349a>3.11 - Implementation status</h1><p>This page summarizes the features supported by different Parquet
implementations.</p><p><em>Note</em>: If you find out of date information, please help us improve the accuracy
of this page by opening an issue or submitting a pull request.</p><h3 id=legend>Legend</h3><p>The value in each box means:</p><ul><li>✅: supported</li><li>❌: not supported</li><li>(R/W): partial reader/writer only support</li><li>(blank): no data</li></ul><p>Implementations:</p><ul><li><a href=https://github.com/apache/arrow/tree/main/cpp/src/parquet>arrow</a> (C++)</li><li><a href=https://github.com/apache/parquet-java>parquet-java</a> (Java)</li><li><a href=https://github.com/apache/ar [...]
Java resources can be build using <code>mvn package</code>. The current stable version should always be available from Maven Central.</p><p>C++ thrift resources can be generated via make.</p><p>Thrift can be also code-genned into any other thrift-supported language.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-47cac26307c77b16f1b9e75c1e46efec>4.3 - Contributing to Parquet-Java</h1><div class=lead>How to contribute to Parquet-Java</div><h2 id=pull-requests>Pull [...]
@@ -900,4 +1790,4 @@ releases on a quarterly basis (Targets months are January, April, July and
October). If a new major version is necessary it will be targetted for the
October release.</p></div><div class=td-content style=page-break-before:always><h1 id=pg-84bc70d5f3df0dfce176f8761d95e996>5 - Resources</h1><div class=lead>Various resources to learn about the Parquet File Format.</div></div><div class=td-content><h1 id=pg-43142841c91c2a4b748c50532b346e58>5.1 - Blog Posts</h1><div class=lead>Blog posts with content about the Parquet File Format.</div><ul><li><p><a href=https://eng.uber.com/cost-efficiency-big-data/>Apache Parquet: Cost Efficiency @ Scale [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/learning-resources/blog-posts/index.html b/output/_print/docs/learning-resources/blog-posts/index.html
index 3bde0c7..f4ca30a 100644
--- a/output/_print/docs/learning-resources/blog-posts/index.html
+++ b/output/_print/docs/learning-resources/blog-posts/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/learning-resources/blog-posts/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/learning-resources/blog-posts/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/learning-resources/blog-posts/>Return to the regular view of this page</a>.</p></div><h1 class=title>Blog Posts</h1><div class=lead>Blog posts with content about the Parquet File Format.</div><ul></ul><div class=content><ul><li><p><a href=https://eng.uber.com/cost-efficiency-big-data/>Apache Parquet: Cost Efficiency @ Scale in Big Data File Format</a></p></li><li><p><a href=https://eng.uber.com/one-stone-t [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/learning-resources/index.html b/output/_print/docs/learning-resources/index.html
index 31947be..67513f7 100644
--- a/output/_print/docs/learning-resources/index.html
+++ b/output/_print/docs/learning-resources/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/learning-resources/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple- [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/learning-resources/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple- [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/learning-resources/>Return to the regular view of this page</a>.</p></div><h1 class=title>Resources</h1><div class=lead>Various resources to learn about the Parquet File Format.</div><ul><li>1: <a href=#pg-43142841c91c2a4b748c50532b346e58>Blog Posts</a></li><ul></ul><li>2: <a href=#pg-f3159f69f9e977c1d1fe8fe51508c564>Presentations</a></li><ul><li>2.1: <a href=#pg-2bab08e87517b89925ede9b22bc0b0a1>Spark Summ [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/learning-resources/presentations/index.html b/output/_print/docs/learning-resources/presentations/index.html
index 6579304..908454b 100644
--- a/output/_print/docs/learning-resources/presentations/index.html
+++ b/output/_print/docs/learning-resources/presentations/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/learning-resources/presentations/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touc [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/learning-resources/presentations/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touc [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/learning-resources/presentations/>Return to the regular view of this page</a>.</p></div><h1 class=title>Presentations</h1><div class=lead>Presentations with content about the Parquet File Format.</div><ul><li>1: <a href=#pg-2bab08e87517b89925ede9b22bc0b0a1>Spark Summit 2020</a></li><li>2: <a href=#pg-a62fdfee9bed68c21f683b06c19ef49a>Hadoop Summit 2014</a></li><li>3: <a href=#pg-76618cb9008bae0854fda78ea6fb [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/overview/index.html b/output/_print/docs/overview/index.html
index 2d8d584..8658ef1 100644
--- a/output/_print/docs/overview/index.html
+++ b/output/_print/docs/overview/index.html
@@ -1,5 +1,5 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/overview/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.p [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/overview/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.p [...]
<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/overview/>Return to the regular view of this page</a>.</p></div><h1 class=title>Overview</h1><div class=lead>All about Parquet.</div><ul><li>1: <a href=#pg-2e5324b574579d3eb78ce816487d9e18>Motivation</a></li></ul><div class=content><p>Apache Parquet is an open source, column-oriented data file format designed for efficient data storage and retrieval.
It provides high performance compression and encoding schemes to handle complex data in bulk and is supported in many programming languages and analytics tools.</p><h3 id=parquet-format-specification>parquet-format (Specification)</h3><p>The <a href=https://github.com/apache/parquet-format>parquet-format</a> repository hosts the official specification of the Parquet file format, defining how data is structured and stored. This specification, along with the <a href=https://github.com/apac [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2019/09/25/2.7.0/index.html b/output/blog/2019/09/25/2.7.0/index.html
index 65ed0d3..1fa4d87 100644
--- a/output/blog/2019/09/25/2.7.0/index.html
+++ b/output/blog/2019/09/25/2.7.0/index.html
@@ -10,10 +10,11 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><meta itemprop=datePublished content="2019-09-25T00:00:00+00:00"><meta itemprop=dateModified content="2024-03-13T22:50:30-04:00"><meta itemprop=wordCount content="42"><meta name=twitter:card content="summary"><meta name=twitter:title content="2.7.0"><meta name=twitter:description content="Github Relea [...]
The latest version of parquet-format is 2.7.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=s [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=s [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/2.7.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=2.7.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcr [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-it [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2020/01/13/2.8.0/index.html b/output/blog/2020/01/13/2.8.0/index.html
index c53a66f..349cb3d 100644
--- a/output/blog/2020/01/13/2.8.0/index.html
+++ b/output/blog/2020/01/13/2.8.0/index.html
@@ -10,10 +10,11 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><meta itemprop=datePublished content="2020-01-13T00:00:00+00:00"><meta itemprop=dateModified content="2024-03-13T22:50:30-04:00"><meta itemprop=wordCount content="42"><meta name=twitter:card content="summary"><meta name=twitter:title content="2.8.0"><meta name=twitter:description content="Github Relea [...]
The latest version of parquet-format is 2.8.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=s [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=s [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/2.8.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=2.8.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcr [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-it [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2021/10/06/1.12.2/index.html b/output/blog/2021/10/06/1.12.2/index.html
index 89d8778..0e62990 100644
--- a/output/blog/2021/10/06/1.12.2/index.html
+++ b/output/blog/2021/10/06/1.12.2/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.12.2</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.12.2</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.12.2.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.12.2" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a class="btn btn-primary disabled"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2022/05/26/1.12.3/ aria-label="Next - 1.12.3" class="btn btn-primary">Nex [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2021/10/06/2.9.0/index.html b/output/blog/2021/10/06/2.9.0/index.html
index 83803cd..f99828a 100644
--- a/output/blog/2021/10/06/2.9.0/index.html
+++ b/output/blog/2021/10/06/2.9.0/index.html
@@ -10,10 +10,11 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><meta itemprop=datePublished content="2021-10-06T00:00:00+00:00"><meta itemprop=dateModified content="2024-03-13T22:50:30-04:00"><meta itemprop=wordCount content="42"><meta name=twitter:card content="summary"><meta name=twitter:title content="2.9.0"><meta name=twitter:description content="Github Relea [...]
The latest version of parquet-format is 2.9.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=s [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=s [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/2.9.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=2.9.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcr [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-it [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2022/05/26/1.12.3/index.html b/output/blog/2022/05/26/1.12.3/index.html
index 3f99bac..32ca1de 100644
--- a/output/blog/2022/05/26/1.12.3/index.html
+++ b/output/blog/2022/05/26/1.12.3/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.12.3</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.12.3</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.12.3.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.12.3" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2021/10/06/1.12.2/ aria-label="Previous - 1.12.2" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2023/04/06/1.13.0/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2023/04/06/1.13.0/index.html b/output/blog/2023/04/06/1.13.0/index.html
index dd54e46..0fa68a2 100644
--- a/output/blog/2023/04/06/1.13.0/index.html
+++ b/output/blog/2023/04/06/1.13.0/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.13.0</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.13.0</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.13.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.13.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2022/05/26/1.12.3/ aria-label="Previous - 1.12.3" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2023/05/18/1.13.1/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2023/05/18/1.13.1/index.html b/output/blog/2023/05/18/1.13.1/index.html
index 4c8b26a..1f4237b 100644
--- a/output/blog/2023/05/18/1.13.1/index.html
+++ b/output/blog/2023/05/18/1.13.1/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.13.1</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.13.1</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.13.1.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.13.1" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2023/04/06/1.13.0/ aria-label="Previous - 1.13.0" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2024/05/07/1.14.0/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2023/11/20/2.10.0/index.html b/output/blog/2023/11/20/2.10.0/index.html
index 05b983c..a8a1a8f 100644
--- a/output/blog/2023/11/20/2.10.0/index.html
+++ b/output/blog/2023/11/20/2.10.0/index.html
@@ -10,10 +10,10 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><meta itemprop=datePublished content="2023-11-20T00:00:00+00:00"><meta itemprop=dateModified content="2024-03-13T22:50:30-04:00"><meta itemprop=wordCount content="40"><meta name=twitter:card content="summary"><meta name=twitter:title content="2.10.0"><meta name=twitter:description content="Github Release Link.
The latest version of parquet-format is 2.10.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integ [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integ [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/2.10.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=2.10.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Releases</a></li><li class=breadcrumb-item><a [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>R [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2024/05/07/1.14.0/index.html b/output/blog/2024/05/07/1.14.0/index.html
index 3d40a1f..c059264 100644
--- a/output/blog/2024/05/07/1.14.0/index.html
+++ b/output/blog/2024/05/07/1.14.0/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.0</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.0</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.14.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.14.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2023/05/18/1.13.1/ aria-label="Previous - 1.13.1" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2024/06/16/1.14.1/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2024/06/16/1.14.1/index.html b/output/blog/2024/06/16/1.14.1/index.html
index f6d4a0f..1a572d4 100644
--- a/output/blog/2024/06/16/1.14.1/index.html
+++ b/output/blog/2024/06/16/1.14.1/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.1</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.1</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.14.1.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.14.1" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2024/05/07/1.14.0/ aria-label="Previous - 1.14.0" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2024/08/26/1.14.2/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2024/08/26/1.14.2/index.html b/output/blog/2024/08/26/1.14.2/index.html
index 51ff7e6..c91a0b7 100644
--- a/output/blog/2024/08/26/1.14.2/index.html
+++ b/output/blog/2024/08/26/1.14.2/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.2</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.2</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.14.2.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.14.2" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2024/06/16/1.14.1/ aria-label="Previous - 1.14.1" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2024/10/05/1.14.3/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2024/10/05/1.14.3/index.html b/output/blog/2024/10/05/1.14.3/index.html
index b7ed3ce..3482d80 100644
--- a/output/blog/2024/10/05/1.14.3/index.html
+++ b/output/blog/2024/10/05/1.14.3/index.html
@@ -22,11 +22,12 @@ Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parq
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central.
Add the following dependency section to your pom.xml:
-<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.3</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9 [...]
+<dependencies> ... <dependency> <groupId>org.apache.parquet</groupId> <artifactId>parquet-avro</artifactId> <version>1.14.3</version> <!-- or latest version --> </dependency> ... </dependencies> Older Releases Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.14.3.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.14.3" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -37,4 +38,4 @@ Add the following dependency section to your pom.xml:
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2024/08/26/1.14.2/ aria-label="Previous - 1.14.2" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2024/11/11/1.14.4/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2024/11/11/1.14.4/index.html b/output/blog/2024/11/11/1.14.4/index.html
index 4970e0d..fedae8f 100644
--- a/output/blog/2024/11/11/1.14.4/index.html
+++ b/output/blog/2024/11/11/1.14.4/index.html
@@ -18,11 +18,12 @@ With the following bugfixes:
GH-3040: DictionaryFilter.canDrop may return false positive result when dict size exceeds 8k GH-3029: Fix EncryptionPropertiesHelper not to use java.nio.file.Path GH-3042: Throw exception in BytesInput To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.13.1.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sha256-IC5 [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.14.4.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.14.4" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -33,4 +34,4 @@ Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2024/10/05/1.14.3/ aria-label="Previous - 1.14.3" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2024/12/02/1.15.0/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2024/12/02/1.15.0/index.html b/output/blog/2024/12/02/1.15.0/index.html
index 0ded699..37acfcf 100644
--- a/output/blog/2024/12/02/1.15.0/index.html
+++ b/output/blog/2024/12/02/1.15.0/index.html
@@ -18,11 +18,12 @@ For the changes, please check out the Release on Github.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.15.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sha256-IC5 [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.15.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.15.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -33,4 +34,4 @@ Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2024/11/11/1.14.4/ aria-label="Previous - 1.14.4" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2025/03/16/1.15.1/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2025/03/16/1.15.1/index.html b/output/blog/2025/03/16/1.15.1/index.html
index 4f86462..c4281c1 100644
--- a/output/blog/2025/03/16/1.15.1/index.html
+++ b/output/blog/2025/03/16/1.15.1/index.html
@@ -18,11 +18,12 @@ For the changes, please check out the Release on Github.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.14.4.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sha256-IC5 [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.15.1.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.15.1" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -33,4 +34,4 @@ Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2024/12/02/1.15.0/ aria-label="Previous - 1.15.0" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2025/05/01/1.15.2/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2025/03/23/2.11.0/index.html b/output/blog/2025/03/23/2.11.0/index.html
index fb318fd..f5193b0 100644
--- a/output/blog/2025/03/23/2.11.0/index.html
+++ b/output/blog/2025/03/23/2.11.0/index.html
@@ -10,10 +10,10 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><meta itemprop=datePublished content="2025-03-23T00:00:00+00:00"><meta itemprop=dateModified content="2025-03-24T15:56:24+08:00"><meta itemprop=wordCount content="40"><meta name=twitter:card content="summary"><meta name=twitter:title content="2.11.0"><meta name=twitter:description content="Github Release Link.
The latest version of parquet-format is 2.11.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integ [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integ [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/2.11.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=2.11.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Releases</a></li><li class=breadcrumb-item><a [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>R [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2025/05/01/1.15.2/index.html b/output/blog/2025/05/01/1.15.2/index.html
index 2cc40bf..c3f3b41 100644
--- a/output/blog/2025/05/01/1.15.2/index.html
+++ b/output/blog/2025/05/01/1.15.2/index.html
@@ -18,11 +18,12 @@ For the changes, please check out the Release on Github.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.14.4.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sha256-IC5 [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.15.2.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.15.2" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -33,4 +34,4 @@ Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2025/03/16/1.15.1/ aria-label="Previous - 1.15.1" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a href=/blog/2025/09/03/1.16.0/ ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2025/08/28/2.12.0/index.html b/output/blog/2025/08/28/2.12.0/index.html
index a09c0f5..e440c04 100644
--- a/output/blog/2025/08/28/2.12.0/index.html
+++ b/output/blog/2025/08/28/2.12.0/index.html
@@ -10,10 +10,10 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><meta itemprop=datePublished content="2025-08-28T00:00:00+00:00"><meta itemprop=dateModified content="2025-08-28T22:19:15+08:00"><meta itemprop=wordCount content="40"><meta name=twitter:card content="summary"><meta name=twitter:title content="2.12.0"><meta name=twitter:description content="Github Release Link.
The latest version of parquet-format is 2.12.0.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integ [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: https://archive.apache.org/dist/parquet/"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integ [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/2.12.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=2.12.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Releases</a></li><li class=breadcrumb-item><a [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>R [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/2025/09/03/1.16.0/index.html b/output/blog/2025/09/03/1.16.0/index.html
index ab86408..4d652d2 100644
--- a/output/blog/2025/09/03/1.16.0/index.html
+++ b/output/blog/2025/09/03/1.16.0/index.html
@@ -18,11 +18,12 @@ For the changes, please check out the Release on Github.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.15.2.
To check the validity of this release, use its:
-Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sha256-IC5 [...]
+Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven central repository The Parquet team publishes its releases to Maven Central."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/1.16.0.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=1.16.0" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/ind [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#downloading-from-the-maven-central-repository>Downloading from the Maven central repository</a></li><li><a href=#older-releases>Older Releases</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-h [...]
...
<dependency>
<groupId>org.apache.parquet</groupId>
@@ -33,4 +34,4 @@ Release manager OpenPGP key OpenPGP signature SHA-512 Downloading from the Maven
</dependencies>
</code></pre><h3 id=older-releases>Older Releases</h3><p>Older releases can be found in the Archives of the Apache Software Foundation: <a href=https://archive.apache.org/dist/parquet/>https://archive.apache.org/dist/parquet/</a></p><ul class="list-unstyled d-flex justify-content-between align-items-center mb-0 pt-5"><li><a href=/blog/2025/05/01/1.15.2/ aria-label="Previous - 1.15.2" class="btn btn-primary"><span class=me-1>←</span>Previous</a></li><li><a class="btn btn-primary disabled" [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/index.html b/output/blog/index.html
index f63ffa8..086f0d9 100644
--- a/output/blog/index.html
+++ b/output/blog/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x18 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x18 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+the+p [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Releases" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current=page>Releases</li></ol></nav><div class=td- [...]
+<a id=print href=/_print/blog/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current [...]
For the changes, please check out the Release on Github.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the …</p><p class=pt-0><a href=/blog/2025/09/03/1.16.0/ aria-label="Read more - 1.16.0">Read more</a></p></div></li><li class=td-blog-posts-list__item><div class=td-blog-posts-list__body><h5 class="mt-0 mb-1"><a href=/blog/2025/08/28/2.12.0/>2.12.0</a></h5><p class="mb-2 mb-md-3"><small class=text-body-secondary>Thursday, August 28, 2025 in parquet-format</small></p><header class=article-meta></he [...]
@@ -33,4 +33,4 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.13.1.
To check the validity of …</p><p class=pt-0><a href=/blog/2024/06/16/1.14.1/ aria-label="Read more - 1.14.1">Read more</a></p></div></li></ul></div><div class=td-blog-posts__pagination><ul class="pagination pagination-default"><li class="page-item disabled"><a aria-disabled=true aria-label=First class=page-link role=button tabindex=-1><span aria-hidden=true>««</span></a></li><li class="page-item disabled"><a aria-disabled=true aria-label=Previous class=page-link role=button tab [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/page/2/index.html b/output/blog/page/2/index.html
index cdc80df..ce23fe7 100644
--- a/output/blog/page/2/index.html
+++ b/output/blog/page/2/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x18 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/><link rel=alternate type=application/rss+xml href=/blog/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x18 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+the+p [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Releases" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current=page>Releases</li></ol></nav><div class=td- [...]
+<a id=print href=/_print/blog/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current [...]
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.13.1.
To check the validity of …</p><p class=pt-0><a href=/blog/2024/05/07/1.14.0/ aria-label="Read more - 1.14.0">Read more</a></p></div></li></ul><div class=h2>Posts in 2023</div><ul class=td-blog-posts-list><li class=td-blog-posts-list__item><div class=td-blog-posts-list__body><h5 class="mt-0 mb-1"><a href=/blog/2023/11/20/2.10.0/>2.10.0</a></h5><p class="mb-2 mb-md-3"><small class=text-body-secondary>Monday, November 20, 2023 in parquet-format</small></p><header class=article-meta></header [...]
@@ -31,4 +31,4 @@ The latest version of parquet-format is 2.7.0.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache …</p><p class=pt-0><a href=/blog/2019/09/25/2.7.0/ aria-label="Read more - 2.7.0">Read more</a></p></div></li></ul></div><div class=td-blog-posts__pagination><ul class="pagination pagination-default"><li class=page-item><a href=/blog/ aria-label=First class=page-link role=button><span aria-hidden=true>««</span></a></li><li class=page-item><a href=/blog/ [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/parquet-format/index.html b/output/blog/parquet-format/index.html
index df2543a..91830e4 100644
--- a/output/blog/parquet-format/index.html
+++ b/output/blog/parquet-format/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/parquet-format/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/parquet-format/><link rel=alternate type=application/rss+xml href=/blog/parquet-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-format/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Parquet%20Format%20Releases" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Releases</a></li><li class="breadcrumb-item a [...]
+<a id=print href=/_print/blog/parquet-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-format/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>R [...]
The latest version of parquet-format is 2.12.0.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older releases can be found in the Archives of the Apache Software Foundation: …</p><p class=pt-0><a href=/blog/2025/08/28/2.12.0/ aria-label="Read more - 2.12.0">Read more</a></p></div></li><li class=td-blog-posts-list__item><div class=td-blog-posts-list__body><h5 class="mt-0 mb-1"><a href=/blog/2025/03/23/2.11.0/>2.11.0</a></h5><p class="mb-2 mb-md-3"><small class=text-body-secondary>Sunday, March 23, 2025 in parquet-format</small>< [...]
@@ -22,4 +22,4 @@ The latest version of parquet-format is 2.7.0.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 Older Releases Older releases can be found in the Archives of the Apache …</p><p class=pt-0><a href=/blog/2019/09/25/2.7.0/ aria-label="Read more - 2.7.0">Read more</a></p></div></li></ul></div><div class=td-blog-posts__pagination></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list> [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/parquet-java/index.html b/output/blog/parquet-java/index.html
index 590a13e..ec2e611 100644
--- a/output/blog/parquet-java/index.html
+++ b/output/blog/parquet-java/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/parquet-java/><link rel=alternate type=application/rss+xml href=/blog/parquet-java/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ico [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/parquet-java/><link rel=alternate type=application/rss+xml href=/blog/parquet-java/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ico [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Parquet-Java%20Releases" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Releases</a></li><li class="breadcrumb-item activ [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Relea [...]
For the changes, please check out the Release on Github.
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the …</p><p class=pt-0><a href=/blog/2025/09/03/1.16.0/ aria-label="Read more - 1.16.0">Read more</a></p></div></li><li class=td-blog-posts-list__item><div class=td-blog-posts-list__body><h5 class="mt-0 mb-1"><a href=/blog/2025/05/01/1.15.2/>1.15.2</a></h5><p class="mb-2 mb-md-3"><small class=text-body-secondary>Thursday, May 01, 2025 in parquet-java</small></p><header class=article-meta></header> [...]
@@ -33,4 +33,4 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.12.3.
To check the validity of …</p><p class=pt-0><a href=/blog/2023/05/18/1.13.1/ aria-label="Read more - 1.13.1">Read more</a></p></div></li></ul></div><div class=td-blog-posts__pagination><ul class="pagination pagination-default"><li class="page-item disabled"><a aria-disabled=true aria-label=First class=page-link role=button tabindex=-1><span aria-hidden=true>««</span></a></li><li class="page-item disabled"><a aria-disabled=true aria-label=Previous class=page-link role=button tab [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/blog/parquet-java/page/2/index.html b/output/blog/parquet-java/page/2/index.html
index e51d689..f9dbe94 100644
--- a/output/blog/parquet-java/page/2/index.html
+++ b/output/blog/parquet-java/page/2/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/parquet-java/><link rel=alternate type=application/rss+xml href=/blog/parquet-java/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ico [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/blog/parquet-java/><link rel=alternate type=application/rss+xml href=/blog/parquet-java/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-ico [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/blog/parquet-java/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/blog/parquet-java?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Parquet-Java%20Releases" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Releases</a></li><li class="breadcrumb-item activ [...]
+<a id=print href=/_print/blog/parquet-java/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5 pe-md-4" role=main><a class=td-rss-button title=RSS href=/blog/parquet-java/index.xml target=_blank rel=noopener><i class="fa-solid fa-rss" aria-hidden=true></i></a><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/blog/>Relea [...]
To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.12.3.
To check the validity of …</p><p class=pt-0><a href=/blog/2023/04/06/1.13.0/ aria-label="Read more - 1.13.0">Read more</a></p></div></li></ul><div class=h2>Posts in 2022</div><ul class=td-blog-posts-list><li class=td-blog-posts-list__item><div class=td-blog-posts-list__body><h5 class="mt-0 mb-1"><a href=/blog/2022/05/26/1.12.3/>1.12.3</a></h5><p class="mb-2 mb-md-3"><small class=text-body-secondary>Thursday, May 26, 2022 in parquet-java</small></p><header class=article-meta></header><p c [...]
@@ -13,4 +13,4 @@ To check the validity of this release, use its:
Release manager OpenPGP key OpenPGP signature SHA-512 The latest version of parquet-java on the previous minor branch is 1.11.2.
To check the validity of …</p><p class=pt-0><a href=/blog/2021/10/06/1.12.2/ aria-label="Read more - 1.12.2">Read more</a></p></div></li></ul></div><div class=td-blog-posts__pagination><ul class="pagination pagination-default"><li class=page-item><a href=/blog/parquet-java/ aria-label=First class=page-link role=button><span aria-hidden=true>««</span></a></li><li class=page-item><a href=/blog/parquet-java/ aria-label=Previous class=page-link role=button><span aria-hidden=true>&# [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/categories/index.html b/output/categories/index.html
index 4beed2c..b2254bf 100644
--- a/output/categories/index.html
+++ b/output/categories/index.html
@@ -1,3 +1,3 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=application/rss+xml href=/categories/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/f [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=application/rss+xml href=/categories/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/f [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/community/index.html b/output/community/index.html
index 280bc6b..65cc808 100644
--- a/output/community/index.html
+++ b/output/community/index.html
@@ -1,4 +1,4 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png si [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png si [...]
<a href=https://www.apache.org>Apache Software Foundation</a>
and is developed and maintained by a contributors from around the world.
We'd love you to join us!</p></span></div></div></section><section class="row td-box td-box--white td-box--height-auto linkbox"><div class="col-xs-12 col-sm-6 col-md-6 col-lg-6"><h2>Learn and Connect</h2><p><b>Development Mailing List</b>
@@ -20,4 +20,4 @@ for agendas of past meetings.</p><ul><li title=Subscribe><a href=https://groups.
Join the Google group to access the calendar invite, agenda, and recordings of past calls.</li></ul></div><div class="col-xs-12 col-sm-6 col-md-6 col-lg-6"><h2>Develop and Contribute</h2><p>If you want to get more involved by contributing to Parquet, please see our
<a href=/docs/contribution-guidelines/>Developer Guide.</a></p></div></section></main><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener href=mailto:dev@parquet.apache.org aria-label="Post to Mailing List"><i clas [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/asf/donate/index.html b/output/docs/asf/donate/index.html
index 2e4f172..e795973 100644
--- a/output/docs/asf/donate/index.html
+++ b/output/docs/asf/donate/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/ASF/donate.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/ASF?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Donate" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Donate</li></ol></nav><div class=td-content><h1>Don [...]
+<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Donate< [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/asf/events/index.html b/output/docs/asf/events/index.html
index a2cebb4..fb3d14d 100644
--- a/output/docs/asf/events/index.html
+++ b/output/docs/asf/events/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/ASF/events.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/ASF?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Events" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Events</li></ol></nav><div class=td-content><h1>Eve [...]
+<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Events< [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/asf/index.html b/output/docs/asf/index.html
index e1b28de..61acf81 100644
--- a/output/docs/asf/index.html
+++ b/output/docs/asf/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png size [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png size [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/ASF/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/ASF?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Apache%20Software%20Foundation%20%28ASF%29" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Apache Software Foundation (ASF)</li></ol></nav><div class=td-content><h1>Apache Software Foundation (ASF)</h1><div class=lead>Apache Sof [...]
+<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Apache Software Foundation (ASF)</li></ol></nav><div class=td-content><h1>Apache Software Fou [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/asf/license/index.html b/output/docs/asf/license/index.html
index ce05934..8d7ab8b 100644
--- a/output/docs/asf/license/index.html
+++ b/output/docs/asf/license/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/ASF/license.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/ASF?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=License" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>License</li></ol></nav><div class=td-content><h1>Li [...]
+<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>License [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/asf/security/index.html b/output/docs/asf/security/index.html
index 3d10775..687632b 100644
--- a/output/docs/asf/security/index.html
+++ b/output/docs/asf/security/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/ASF/security.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/ASF?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Security" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Security</li></ol></nav><div class=td-content><h1>S [...]
+<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Securit [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/asf/sponsor/index.html b/output/docs/asf/sponsor/index.html
index 382406c..9fa5ba3 100644
--- a/output/docs/asf/sponsor/index.html
+++ b/output/docs/asf/sponsor/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/ASF/sponsor.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/ASF?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+t [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Sponsor" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Sponsor</li></ol></nav><div class=td-content><h1>Sp [...]
+<a id=print href=/_print/docs/asf/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/asf/>Apache Software Foundation (ASF)</a></li><li class="breadcrumb-item active" aria-current=page>Sponsor [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/concepts/index.html b/output/docs/concepts/index.html
index 7a861eb..25611ff 100644
--- a/output/docs/concepts/index.html
+++ b/output/docs/concepts/index.html
@@ -1,8 +1,9 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/concepts/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x18 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/concepts/><link rel=alternate type=application/rss+xml href=/docs/concepts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x18 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Concepts/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Concepts?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Concepts" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/concepts/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#unit-of-parallelization>Unit of parallelization</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page> [...]
+<a id=print href=/_print/docs/concepts/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><a href=#unit-of-parallelization>Unit of parallelization</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Concepts</li></ol></nav><div class=td-content><h1>Concepts</h1> [...]
unchanged for describing this file format. The file format is
designed to work well on top of HDFS.</p></li><li><p><em>File</em>: A HDFS file that must include the metadata for the file.
It does not need to actually contain the data.</p></li><li><p><em>Row group</em>: A logical horizontal partitioning of the data into rows.
@@ -14,4 +15,4 @@ be multiple page types which are interleaved in a column chunk.</p></li></ul><p>
contains exactly one column chunk per column. Column chunks contain one or
more pages.</p><h2 id=unit-of-parallelization>Unit of parallelization</h2><ul><li>MapReduce - File/Row Group</li><li>IO - Column chunk</li><li>Encoding/Compression - Page</li></ul><div class=section-index></div><div class=td-page-meta__lastmod>Last modified March 8, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b3b81ce3e9f9e6f25b41f463577976628515384a>Update to new website (b3b81ce)</a></div></div></main></div></div><footer class="td-footer row d-print-n [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/contribution-guidelines/building/index.html b/output/docs/contribution-guidelines/building/index.html
index d9597f2..a0480e9 100644
--- a/output/docs/contribution-guidelines/building/index.html
+++ b/output/docs/contribution-guidelines/building/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Contribution%20Guidelines/building.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Contribution%20Guidelines?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+s [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Building%20Parquet" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contribution-guidelines/>Developer Guide</a></li><li class="breadcrumb-item active" aria-current=page>Building Parquet</li></ol></ [...]
+<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contribution-guidelines/>Developer Guide</a></li><li class="breadcrumb-item active" ar [...]
Java resources can be build using <code>mvn package</code>. The current stable version should always be available from Maven Central.</p><p>C++ thrift resources can be generated via make.</p><p>Thrift can be also code-genned into any other thrift-supported language.</p><div class=td-page-meta__lastmod>Last modified March 24, 2022: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/3563721676b364b767058a953f2bcc3e2c0c4b09>Final Squash (3563721)</a></div></div></main [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/contribution-guidelines/contributing/index.html b/output/docs/contribution-guidelines/contributing/index.html
index 55b894a..d542a69 100644
--- a/output/docs/contribution-guidelines/contributing/index.html
+++ b/output/docs/contribution-guidelines/contributing/index.html
@@ -1,8 +1,9 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Contribution%20Guidelines/contributing.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Contribution%20Guidelines?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+s [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Contributing%20to%20Parquet-Java" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#pull-requests>Pull Requests</a></li><li><a href=#committers>Committers</a><ul><li><a href=#merging-a-pull-request>Merging a Pull Request</a></li><li><a href=#semantic-versioning>Semantic versioning</a></li><li><a href=#tracking-issues-using-milestones>Tracking issues using Milestones</a></li><li><a href=#ma [...]
+<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><a href=#pull-requests>Pull Requests</a></li><li><a href=#committers>Committers</a><ul><li><a href=#merging-a-pull-request>Merging a Pull Request</a></li><li><a href=#semantic-versioning>Semantic versioning</a></li><li><a href=#tracking-issues-using-milestones>Tracking issues using Milestones</a></li><li><a href=#maintenance-branches>Maintenance branches</a></li></ul></li><li><a href=#website [...]
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * @param c the current class
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * @return the corresponding logger
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * @deprecated will be removed in 2.0.0; use org.slf4j.LoggerFactory instead.
@@ -21,4 +22,4 @@
job in the <a href=https://github.com/apache/parquet-site/blob/staging/.github/workflows/deploy.yml>deployment workflow</a> will be run, populating the <code>asf-staging</code> branch on this repo with the necessary files.</li></ol><p><strong>Do not directly edit the <code>asf-staging</code> branch of this repo</strong></p><h4 id=production>Production</h4><p>To make a change to the <code>production</code> version of the website:</p><ol><li>Make a PR against the <code>production</code> br [...]
job in the <a href=https://github.com/apache/parquet-site/blob/production/.github/workflows/deploy.yml>deployment workflow</a> will be run, populating the <code>asf-site</code> branch on this repo with the necessary files.</li></ol><p><strong>Do not directly edit the <code>asf-site</code> branch of this repo</strong></p><div class=td-page-meta__lastmod>Last modified November 12, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/a506efc9d2d708e86709841311f322 [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/contribution-guidelines/index.html b/output/docs/contribution-guidelines/index.html
index ef855e6..ae27d1c 100644
--- a/output/docs/contribution-guidelines/index.html
+++ b/output/docs/contribution-guidelines/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/contribution-guidelines/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/fa [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/contribution-guidelines/><link rel=alternate type=application/rss+xml href=/docs/contribution-guidelines/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/fa [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Contribution%20Guidelines/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Contribution%20Guidelines?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+s [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Developer%20Guide" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Developer Guide</li></ol></nav><div class=td-content><h1>Developer Guide</h1><div class=lead>All developer resources [...]
+<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Developer Guide</li></ol></nav><div class=td-content><h1>Developer Guide< [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/contribution-guidelines/releasing/index.html b/output/docs/contribution-guidelines/releasing/index.html
index f277acd..3313543 100644
--- a/output/docs/contribution-guidelines/releasing/index.html
+++ b/output/docs/contribution-guidelines/releasing/index.html
@@ -1,8 +1,9 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Contribution%20Guidelines/releasing.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Contribution%20Guidelines?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+s [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Releasing%20Parquet-Java" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#setup>Setup</a></li><li><a href=#release-process>Release process</a></li><li><a href=#5-prepare-the-pre-release>5. Prepare the pre-release</a></li><li><a href=#publishing-after-the-vote-passes>Publishing after the vote passes</a></li><li><a href=#5-add-the-release-to-github>5. Add the release to Git [...]
+<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#setup>Setup</a></li><li><a href=#release-process>Release process</a></li><li><a href=#5-prepare-the-pre-release>5. Prepare the pre-release</a></li><li><a href=#publishing-after-the-vote-passes>Publishing after the vote passes</a></li><li><a href=#5-add-the-release-to-github>5. Add the release to GitHub</a></li><li><a href=#release-cadence>Release Cadence</a></li></ul></li></u [...]
</span></span></code></pre></div><p>If you have problems, read the <a href=https://www.apache.org/dev/publishing-maven-artifacts.html>publishing Maven artifacts documentation</a>.</p><h3 id=release-process>Release process</h3><p>Parquet uses the maven-release-plugin to tag a release and push binary artifacts to staging in Nexus. Once maven completes the release, the official source tarball is built from the tag.</p><h4 id=0-before-you-start-the-release-process>0. Before you start the rel [...]
</span></span></code></pre></div><p>This runs maven’s release prepare with a consistent tag name. After this step, the release tag will exist in the git repository.</p><p>If this step fails, you can roll back the changes by running these commands.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-sh data-lang=sh><span style=display:flex><span>find ./ -type f -name <span style=color:#4e9a06>'*.releaseBa [...]
</span></span><span style=display:flex><span>find ./ -type f -name <span style=color:#4e9a06>'pom.xml'</span> -exec git checkout <span style=color:#ce5c00;font-weight:700>{}</span> <span style=color:#4e9a06>\;</span>
@@ -66,4 +67,4 @@ releases on a quarterly basis (Targets months are January, April, July and
October). If a new major version is necessary it will be targetted for the
October release.</p><div class=td-page-meta__lastmod>Last modified March 14, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/04b6d1e2a8bc3694e1e81d8cef6474fdf50ecbfe>MINOR: Skip tests when uploading JARs (#87) (04b6d1e)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__lin [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/contribution-guidelines/sub-projects/index.html b/output/docs/contribution-guidelines/sub-projects/index.html
index d1f4d4f..1323120 100644
--- a/output/docs/contribution-guidelines/sub-projects/index.html
+++ b/output/docs/contribution-guidelines/sub-projects/index.html
@@ -6,10 +6,10 @@ The parquet-cpp project is a C++ library to read-write Parquet files. It is part
The parquet-java project is a Java library to read and write Parquet files. It consists of multiple sub-modules, which implement the core components of reading and writing a nested, column-oriented data stream, to and from the Parquet format, along with Hadoop Input/Output Formats, Pig loaders, and other Java-based utilities for interacting with Parquet.
The parquet-cpp project is a C++ library to read-write Parquet files. It is part of the Apache Arrow C++ implementation, with bindings to Python, R, Ruby and C/GLib."><meta itemprop=dateModified content="2025-01-17T00:14:09+08:00"><meta itemprop=wordCount content="166"><meta name=twitter:card content="summary"><meta name=twitter:title content="Sub-Projects"><meta name=twitter:description content="The parquet-format project contains format specifications and Thrift definitions of metadata [...]
The parquet-java project is a Java library to read and write Parquet files. It consists of multiple sub-modules, which implement the core components of reading and writing a nested, column-oriented data stream, to and from the Parquet format, along with Hadoop Input/Output Formats, Pig loaders, and other Java-based utilities for interacting with Parquet.
-The parquet-cpp project is a C++ library to read-write Parquet files. It is part of the Apache Arrow C++ implementation, with bindings to Python, R, Ruby and C/GLib."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sh [...]
+The parquet-cpp project is a C++ library to read-write Parquet files. It is part of the Apache Arrow C++ implementation, with bindings to Python, R, Ruby and C/GLib."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sh [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Contribution%20Guidelines/sub-projects.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Contribution%20Guidelines?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+s [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Sub-Projects" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contribution-guidelines/>Developer Guide</a></li><li class="breadcrumb-item active" aria-current=page>Sub-Projects</li></ol></nav> [...]
+<a id=print href=/_print/docs/contribution-guidelines/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/contribution-guidelines/>Developer Guide</a></li><li class="breadcrumb-item active" ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/_print/docs/asf/index.html b/output/docs/file-format/binaryprotocolextensions/index.html
similarity index 89%
copy from output/_print/docs/asf/index.html
copy to output/docs/file-format/binaryprotocolextensions/index.html
index 99f0617..73a4e33 100644
--- a/output/_print/docs/asf/index.html
+++ b/output/docs/file-format/binaryprotocolextensions/index.html
@@ -1,4 +1,74 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=canonical type=text/html href=/docs/asf/><link rel=alternate type=application/rss+xml href=/docs/asf/index.xml><meta name=robots content="noindex, nofollow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=1 [...]
-<a href=# onclick="return print(),!1">Click here to print</a>.</p><p><a href=/docs/asf/>Return to the regular view of this page</a>.</p></div><h1 class=title>Apache Software Foundation (ASF)</h1><div class=lead>Apache Software Foundation</div><ul><li>1: <a href=#pg-57942ee0c37ed732b8a36ad6da469b9b>License</a></li><li>2: <a href=#pg-4a41ca42a224dc588a84f6aceb800983>Security</a></li><li>3: <a href=#pg-18eeebb45e0fdfd01843649980749ac6>Sponsor</a></li><li>4: <a href=#pg-f237f8bebf4f33fc4aedb [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Existing readers will ignore these extensions without any modifications Existing readers will ignore the extension bytes with little processing overhead The content of the extension is freeform and can be encoded in any format. This format is not restricted to Thrift. Extensions can be appended to existing Thrift serialized structs without requiring Thrift libraries for manipulation (or changes to the thrift IDL). Because only one field-id is reserved the extension bytes themselves requi [...]
+Existing readers will ignore these extensions without any modifications Existing readers will ignore the extension bytes with little processing overhead The content of the extension is freeform and can be encoded in any format. This format is not restricted to Thrift. Extensions can be appended to existing Thrift serialized structs without requiring Thrift libraries for manipulation (or changes to the thrift IDL). Because only one field-id is reserved the extension bytes themselves requi [...]
+Existing readers will ignore these extensions without any modifications Existing readers will ignore the extension bytes with little processing overhead The content of the extension is freeform and can be encoded in any format. This format is not restricted to Thrift. Extensions can be appended to existing Thrift serialized structs without requiring Thrift libraries for manipulation (or changes to the thrift IDL). Because only one field-id is reserved the extension bytes themselves requi [...]
+Existing readers will ignore these extensions without any modifications Existing readers will ignore the extension bytes with little processing overhead The content of the extension is freeform and can be encoded in any format. This format is not restricted to Thrift. Extensions can be appended to existing Thrift serialized structs without requiring Thrift libraries for manipulation (or changes to the thrift IDL). Because only one field-id is reserved the extension bytes themselves requi [...]
+<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/binaryprotocolextensions.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
+<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Binary Proto [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(M) encoded size of the extension
+M bytes | extension bytes
+1 byte | \0 (thrift stop field)
+</code></pre><p>The choice to reserve only one field-id has an additional (and frankly unintended) property. It creates scarcity in the extension space and disincentivizes vendors from keeping their extensions private. As a vendor having an extension means one cannot use it in tandem with other extensions from other vendors even if such extensions are publicly known. The easiest path of interoperability and ability to further experiment is to push an extension through standardization and [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(K+28) encoded size of the extension
+K bytes | Flatbuffers representation (v0) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+16 bytes | some-UUID
+1 byte | \0 (thrift stop field)
+4 bytes | PAR1
+</code></pre><p>some-UUID is some UUID picked for this extension and it is used throughout (possibly internal) experimentation. It is put at the end to allow detection of the extension when parsed in reverse. The little-endian sizes and crc32s are also to the end to facilitate efficient parsing the footer in reverse without requiring parsing the Thrift compact protocol that precedes it.</p><p>At some point the experiments conclude and the extension shared publicly with the community. The [...]
+4 bytes | 08 FF FF 01 (long form header for 32767: binary)
+1-5 bytes | ULEB128(K+28) encoded size of the extension
+K bytes | Flatbuffers representation (v1) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+16 bytes | some-other-UUID
+1 byte | \0 (thrift stop field)
+4 bytes | PAR1
+</code></pre><p>After the migration period, the end of the Parquet file may look like this:</p><pre><code>K bytes | Flatbuffers representation (v1) of FileMetaData
+4 bytes | little-endian crc32(flatbuffer)
+4 bytes | little-endian size(flatbuffer)
+4 bytes | little-endian crc32(size(flatbuffer))
+4 bytes | PAR3
+</code></pre><p>In this example, we see several design decisions for the extension at play:</p><ul><li>There is a new some-other-UUID for the accepted change to the standard and now the Thrift <code>FileMetaData</code> cannot be extended itself.</li><li>The length of the footer and the crc32 of the length itself, guarantees that new readers will not overshoot reading bytes in case of corrupt bits in these critical 8 bytes of the file.</li><li>The crc32 of the flatbuffer representation en [...]
+ | | Column b (new encoding)
+ | | Column c (new encoding)
+R bytes | Row Group | Column a
+ | 0 | Column d
+ | | Column b (old encoding)
+ | | Column c (old encoding)
+ | | FileMetaData
+ | | ColumnMetaData: a
+ | | ColumnMetaData: b
+F bytes | | <extension-blob with offsets to new encoding>
+ | | ColumnMetaData: c
+ | | <extension-blob with offsets to new encoding>
+ | | ColumnMetaData: d
+4 bytes | PAR1
+</code></pre><p>The custom reader is compiled with thrift IDL with a binary for field with id 32767. This is done to become extension aware and inspect the extension bytes looking for the UUID disambiguator. If that’s found it decodes the offsets from the rest of the bytes and reads the region of the file containing the new encoding.</p><p>If/when the encoding is ratified, it is added to the official specification as an additional type in <code>Encodings</code> at which point the extensi [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>while</span> <span style=color:#000;font-weight:700>(</span><span style=color:#204a87>true</span><span style=color:#000;font-weight:700>)</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>uint8_t</span> <span style=color:#000>c</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700>&</span> <span style=color:#0000cf;font-weight:700>0x7F</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700><</span> <span style=color:#0000cf;font-weight:700>0x80</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>out</span><span style=color:#ce5c00;font-weight:700>-></span><span style=co [...]
+</span></span><span style=display:flex><span> <span style=color:#000>out</span><span style=color:#ce5c00;font-weight:700>-></span><span style=color:#000>push_back</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>c</span> <span style=color:#ce5c00;font-weight:700>+</span> <span style=color:#0000cf;font-weight:700>0x80</span><span style=color:#000;font-weight:700>);</span>
+</span></span><span style=display:flex><span> <span style=color:#000>x</span> <span style=color:#ce5c00;font-weight:700>>>=</span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>};</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#000>std</span><span style=color:#ce5c00;font-weight:700>::</span><span style=color:#000>string</span> <span style=color:#000>AppendExtension</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>std</span><span style=color:#ce5c00;font-weight:700>::</span><span style=color:#000>string</span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87;fon [...]
+</span></span><span style=display:flex><span> <span style=color:#000>assert</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>back</span><span style=color:#000;font-weight:700>()</span> <span style=color:#ce5c00;font-weight:700>==</span> <span style=color:#4e9a06>'\x00'</span><span style=color:#000;font-weight:700>);</span> <span style=color:#8f5902;font-style:italic>// [...]
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>back</span><span style=color:#000;font-weight:700>()</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#4e9a06>'\x08'</span><span style=color:#000;font-weight:700>;</span> <span style=color:#8f5902;font-style:italic>// replace stop field with [...]
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AppendUleb</span><span style=color:#000;font-weight:700>(</span><span style=color:#0000cf;font-weight:700>32767</span><span style=color:#000;font-weight:700>,</span> <span style=color:#ce5c00;font-weight:700>&</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>);</span> <span style=color:#8f5902;font-style:italic>// field-id
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#000>AppendUleb</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>ext</span><span style=color:#000;font-weight:700>.</span><span style=color:#000>size</span><span style=color:#000;font-weight:700>(),</span> <span style=color:#ce5c00;font-weight:700>&</span><span style=color:#000>thrift</span><span style=color:#000;font-weight:700>);</span>
+</span></span><span style=display:flex><span> <span style=color:#000>thrift</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#000>ext</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span> <span style=color:#000>thrift</span> <span style=color:#ce5c00;font-weight:700>+=</span> <span style=color:#4e9a06>'\x00'</span><span style=color:#000;font-weight:700>;</span> <span style=color:#8f5902;font-style:italic>// add the stop field
+</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic></span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>thrift</span><span style=color:#000;font-weight:700>;</span>
+</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
+</span></span></code></pre></div><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-lis [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/bloomfilter/index.html b/output/docs/file-format/bloomfilter/index.html
index 55a1a06..c9a1e38 100644
--- a/output/docs/file-format/bloomfilter/index.html
+++ b/output/docs/file-format/bloomfilter/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/bloomfilter.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Bloom%20Filter" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#problem-statement>Problem statement</a></li><li><a href=#goal>Goal</a></li><li><a href=#technical-approach>Technical Approach</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcr [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Bloom Filter [...]
pushdown. Statistics include minimum and maximum value, which can be used to filter out
values not in the range. Dictionaries are more specific, and readers can filter out values
that are between min and max but not in the dictionary. However, when there are too many
@@ -134,7 +134,7 @@ significant 32 bits.</p><pre tabindex=0><code>void filter_insert(SBBF filter, un
block b = filter.getBlock(i);
return block_check(b, (unsigned int32)x)
}
-</code></pre><p>The use of blocks is from Putze et al.’s <a href=https://www.cs.amherst.edu/~ccmcgeoch/cs34/papers/cacheefficientbloomfilters-jea.pdf>Cache-, Hash- and
+</code></pre><p>The use of blocks is from Putze et al.’s <a href=http://algo2.iti.kit.edu/documents/cacheefficientbloomfilters-jea.pdf>Cache-, Hash- and
Space-Efficient Bloom
filters</a></p><p>To use an SBBF for values of arbitrary Parquet types, we apply a hash
function to that value - at the time of writing,
@@ -212,8 +212,8 @@ struct ColumnMetaData {
}
</code></pre><p>The Bloom filters are grouped by row group and with data for each column in the same order as the file schema.
The Bloom filter data can be stored before the page indexes after all row groups. The file layout looks like:
-<img src=/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
-<img src=/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
+<img src=/parquet-format/doc/images/FileLayoutBloomFilter2.png alt="File Layout - Bloom filter footer"></p><p>Or it can be stored between row groups, the file layout looks like:
+<img src=/parquet-format/doc/images/FileLayoutBloomFilter1.png alt="File Layout - Bloom filter footer"></p><h4 id=encryption>Encryption</h4><p>In the case of columns with sensitive data, the Bloom filter exposes a subset of sensitive
information such as the presence of value. Therefore the Bloom filter of columns with sensitive
data should be encrypted with the column key, and the Bloom filter of other (not sensitive) columns
do not need to be encrypted.</p><p>Bloom filters have two serializable modules - the PageHeader thrift structure (with its internal
@@ -223,6 +223,6 @@ serialized Bitset.</p><p>For Bloom filters in sensitive columns, each of the two
serialization, and then written to the file. The encryption will be performed using the AES GCM
cipher, with the same column key, but with different AAD module types - “BloomFilter Header” (8)
and “BloomFilter Bitset” (9). The length of the encrypted buffer is written before the buffer, as
-described in the Parquet encryption specification.</p><div class=td-page-meta__lastmod>Last modified March 11, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/e79b30489c6bd50f0829a5f2b87f4a26f5e4af05>Fix typos (#46) (e79b304)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-foote [...]
+described in the Parquet encryption specification.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/configurations/index.html b/output/docs/file-format/configurations/index.html
index 22eb43e..bbe6a66 100644
--- a/output/docs/file-format/configurations/index.html
+++ b/output/docs/file-format/configurations/index.html
@@ -1,8 +1,9 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/configurations.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Configurations" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#row-group-size>Row Group Size</a></li><li><a href=#data-page--size>Data Page Size</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></l [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#row-group-size>Row Group Size</a></li><li><a href=#data-page--size>Data Page Size</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format [...]
possible to do larger sequential IO. Larger groups also require more buffering in
the write path (or a two pass write). We recommend large row groups (512MB - 1GB).
Since an entire row group might need to be read, we want it to completely fit on
@@ -14,4 +15,4 @@ incur less space overhead (less page headers) and potentially less parsing overh
(processing headers). Note: for sequential scans, it is not expected to read a page
at a time; this is not the IO chunk. We recommend 8KB for page sizes.</p><div class=td-page-meta__lastmod>Last modified March 8, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b3b81ce3e9f9e6f25b41f463577976628515384a>Update to new website (b3b81ce)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/checksumming/index.html b/output/docs/file-format/data-pages/checksumming/index.html
index 90b1962..e7a21fc 100644
--- a/output/docs/file-format/data-pages/checksumming/index.html
+++ b/output/docs/file-format/data-pages/checksumming/index.html
@@ -1,10 +1,10 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/checksumming.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Checksumming" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/data-pages/>Data Pages</a></li><li class="brea [...]
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/da [...]
at the HDFS file level, to better support single row lookups. Checksums are calculated
using the standard CRC32 algorithm - as used in e.g. GZip - on the serialized binary
representation of a page (not including the page header itself).</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/columnchunks/index.html b/output/docs/file-format/data-pages/columnchunks/index.html
index cb1b000..b5659a2 100644
--- a/output/docs/file-format/data-pages/columnchunks/index.html
+++ b/output/docs/file-format/data-pages/columnchunks/index.html
@@ -2,11 +2,11 @@
A column chunk might be partly or completely dictionary encoded. It means that dictionary indexes are saved in the data pages instead of the actual values. The actual values are stored in the dictionary page. See details in Encodings.md. The dictionary page must be placed at the first position of the column chunk. At most one dictionary page can be placed in a column chunk."><meta property="og:url" content="/docs/file-format/data-pages/columnchunks/"><meta property="og:site_name" content [...]
A column chunk might be partly or completely dictionary encoded. It means that dictionary indexes are saved in the data pages instead of the actual values. The actual values are stored in the dictionary page. See details in Encodings.md. The dictionary page must be placed at the first position of the column chunk. At most one dictionary page can be placed in a column chunk."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section [...]
A column chunk might be partly or completely dictionary encoded. It means that dictionary indexes are saved in the data pages instead of the actual values. The actual values are stored in the dictionary page. See details in Encodings.md. The dictionary page must be placed at the first position of the column chunk. At most one dictionary page can be placed in a column chunk."><meta itemprop=dateModified content="2024-01-14T20:32:15+08:00"><meta itemprop=wordCount content="145"><meta name= [...]
-A column chunk might be partly or completely dictionary encoded. It means that dictionary indexes are saved in the data pages instead of the actual values. The actual values are stored in the dictionary page. See details in Encodings.md. The dictionary page must be placed at the first position of the column chunk. At most one dictionary page can be placed in a column chunk."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=styl [...]
+A column chunk might be partly or completely dictionary encoded. It means that dictionary indexes are saved in the data pages instead of the actual values. The actual values are stored in the dictionary page. See details in Encodings.md. The dictionary page must be placed at the first position of the column chunk. At most one dictionary page can be placed in a column chunk."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=styl [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/columnchunks.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Column%20Chunks" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/data-pages/>Data Pages</a></li><li class="brea [...]
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/da [...]
header and readers can skip over pages they are not interested in. The data for the
page follows the header and can be compressed and/or encoded. The compression and
encoding is specified in the page metadata.</p><p>A column chunk might be partly or completely dictionary encoded. It means that
@@ -17,4 +17,4 @@ most one dictionary page can be placed in a column chunk.</p><p>Additionally, fi
skip pages more efficiently. See PageIndex.md for details and
the reasoning behind adding these to the format.</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-l [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/compression/index.html b/output/docs/file-format/data-pages/compression/index.html
index 4c5eabe..79c89aa 100644
--- a/output/docs/file-format/data-pages/compression/index.html
+++ b/output/docs/file-format/data-pages/compression/index.html
@@ -1,19 +1,23 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Overview Parquet allows the data block inside dictionary pages and data pages to be compressed for better space efficiency. The Parquet format supports several compression codecs covering different areas in the compression ratio / processing cost spectrum.
The detailed specifications of compression codecs are maintained externally by their respective authors or maintainers, which we reference hereafter.
-For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><meta property="og:url" content="/docs/file-format/data-pages/compression/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Compressi [...]
+For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><meta property="og:url" content="/docs/file-format/data-pages/compression/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Parquet"> [...]
+Overview Parquet allows the data block inside dictionary pages and data pages to be compressed for better space efficiency. The Parquet format supports several compression codecs covering different areas in the compression ratio / processing cost spectrum.
The detailed specifications of compression codecs are maintained externally by their respective authors or maintainers, which we reference hereafter.
-For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time [...]
+For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time [...]
+Overview Parquet allows the data block inside dictionary pages and data pages to be compressed for better space efficiency. The Parquet format supports several compression codecs covering different areas in the compression ratio / processing cost spectrum.
The detailed specifications of compression codecs are maintained externally by their respective authors or maintainers, which we reference hereafter.
-For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><meta itemprop=dateModified content="2024-03-11T22:11:10+01:00"><meta itemprop=wordCount content="379"><meta name=twitter:card content="summary"><meta name=twitter:titl [...]
+For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop=wordCount content="393"><meta name=twitter:card content="summary"><meta name=twitter:titl [...]
+Overview Parquet allows the data block inside dictionary pages and data pages to be compressed for better space efficiency. The Parquet format supports several compression codecs covering different areas in the compression ratio / processing cost spectrum.
The detailed specifications of compression codecs are maintained externally by their respective authors or maintainers, which we reference hereafter.
-For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQ [...]
+For all compression codecs except the deprecated LZ4 codec, the raw data of a (data or dictionary) page is fed as-is to the underlying compression library, without any additional framing or padding. The information required for precise allocation of compressed and decompressed buffers is written in the PageHeader struct."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/compression.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Compression" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#overview>Overview</a></li><li><a href=#codecs>Codecs</a><ul><li><a href=#uncompressed>UNCOMPRESSED</a></li><li><a href=#snappy>SNAPPY</a></li><li><a href=#gzip>GZIP</a></li><li><a href=#lzo>LZO</a></li><li><a href=#brotli>BROTLI</a></li><li><a href=#lz4>LZ4</a></li><li><a href=#zstd>ZSTD</a></li><li><a href= [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/da [...]
be compressed for better space efficiency. The Parquet format supports
-several compression covering different areas in the compression ratio /
-processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
+several compression codecs covering different areas in the compression
+ratio / processing cost spectrum.</p><p>The detailed specifications of compression codecs are maintained externally
by their respective authors or maintainers, which we reference hereafter.</p><p>For all compression codecs except the deprecated <code>LZ4</code> codec, the raw data
of a (data or dictionary) page is fed <em>as-is</em> to the underlying compression
library, without any additional framing or padding. The information required
@@ -43,6 +47,6 @@ when implementing this format, the implementation provided by the
<a href=https://facebook.github.io/zstd/>Zstandard compression library</a>
is authoritative.</p><h3 id=lz4_raw>LZ4_RAW</h3><p>A codec based on the <a href=https://github.com/lz4/lz4/blob/dev/doc/lz4_Block_format.md>LZ4 block format</a>.
If any ambiguity arises when implementing this format, the implementation
-provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p><div class=td-page-meta__lastmod>Last modified March 11, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/e79b30489c6bd50f0829a5f2b87f4a26f5e4af05>Fix typos (#46) (e79b304)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class= [...]
+provided by the <a href=https://www.lz4.org/>LZ4 compression library</a> is authoritative.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__l [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/encodings/index.html b/output/docs/file-format/data-pages/encodings/index.html
index d887213..7a8266a 100644
--- a/output/docs/file-format/data-pages/encodings/index.html
+++ b/output/docs/file-format/data-pages/encodings/index.html
@@ -1,21 +1,25 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Plain: (PLAIN = 0) Supported Types: all
This is the plain encoding that must be supported for types. It is intended to be the simplest encoding. Values are encoded back to back.
-The plain encoding is used whenever a more efficient encoding cannot be used. It stores the data in the following format:
-BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><meta property="og:url" content="/docs/file-fo [...]
+The plain encoding is used whenever a more efficient encoding can not be used. It stores the data in the following format:
+BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><meta property="og:url" content="/docs/file-fo [...]
+Plain: (PLAIN = 0) Supported Types: all
This is the plain encoding that must be supported for types. It is intended to be the simplest encoding. Values are encoded back to back.
-The plain encoding is used whenever a more efficient encoding cannot be used. It stores the data in the following format:
-BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><meta property="og:locale" content="en"><meta [...]
+The plain encoding is used whenever a more efficient encoding can not be used. It stores the data in the following format:
+BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><meta property="og:locale" content="en"><meta [...]
+Plain: (PLAIN = 0) Supported Types: all
This is the plain encoding that must be supported for types. It is intended to be the simplest encoding. Values are encoded back to back.
-The plain encoding is used whenever a more efficient encoding cannot be used. It stores the data in the following format:
-BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><meta itemprop=dateModified content="2025-11-1 [...]
+The plain encoding is used whenever a more efficient encoding can not be used. It stores the data in the following format:
+BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><meta itemprop=dateModified content="2025-12-0 [...]
+Plain: (PLAIN = 0) Supported Types: all
This is the plain encoding that must be supported for types. It is intended to be the simplest encoding. Values are encoded back to back.
-The plain encoding is used whenever a more efficient encoding cannot be used. It stores the data in the following format:
-BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><link rel=preload href=/scss/main.min.202e73a8 [...]
+The plain encoding is used whenever a more efficient encoding can not be used. It stores the data in the following format:
+BOOLEAN: Bit Packed, LSB first INT32: 4 bytes little endian INT64: 8 bytes little endian INT96: 12 bytes little endian (deprecated) FLOAT: 4 bytes IEEE little endian DOUBLE: 8 bytes IEEE little endian BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array FIXED_LEN_BYTE_ARRAY: the bytes contained in the array For native types, this outputs the data as little endian. Floating point types are encoded in IEEE."><link rel=preload href=/scss/main.min.46bcfa67 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/encodings.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Encodings" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#plain-plain--0>Plain: (PLAIN = 0)</a></li><li><a href=#dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</a></li><li><a href=#run-length-encoding--bit-packing-hybrid-rle--3>Run Length Encoding / Bit-Packing Hybrid (RLE = 3) [...]
-intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding cannot be used. It
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/da [...]
+intended to be the simplest encoding. Values are encoded back to back.</p><p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p><ul><li>BOOLEAN: <a href=/docs/file-format/data-pages/encodings/#BITPACKED>Bit Packed</a>, LSB first</li><li>INT32: 4 bytes little endian</li><li>INT64: 8 bytes little endian</li><li>INT96: 12 bytes little endian (deprecated)</li><li>FLOAT: 4 bytes IEEE little endian</li><li>DOUBLE: 8 bytes IEEE little endian</li><li>BYTE_ARRAY: length in 4 bytes little endian followed by the bytes contained in the array</li><li>FIXED_LEN_BYTE_ARRAY: the bytes [...]
point types are encoded in IEEE.</p><p>For the byte array type, it encodes the length as a 4 byte little
endian, followed by the bytes.</p><h3 id=dictionary-encoding-plain_dictionary--2-and-rle_dictionary--8>Dictionary Encoding (PLAIN_DICTIONARY = 2 and RLE_DICTIONARY = 8)</h3><p>The dictionary encoding builds a dictionary of values encountered in a given column. The
@@ -42,8 +46,7 @@ repeated-value := value that is repeated, using a fixed-width of round-up-to-nex
</code></pre><ol><li><p>The bit-packing here is done in a different order than the one in the <a href=/docs/file-format/data-pages/encodings/#BITPACKED>deprecated bit-packing</a> encoding.
The values are packed from the least significant bit of each byte to the most significant bit,
though the order of the bits in each value remains in the usual order of most significant to least
-significant. An example of the encoding is presented below. For comparison, the same case is shown
-in the example of the deprecated bit-packing encoding in the next section.</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
+significant. For example, to pack the same values as the example in the deprecated encoding above:</p><p>The numbers 1 through 7 using bit width 3:</p><pre tabindex=0><code>dec value: 0 1 2 3 4 5 6 7
bit value: 000 001 010 011 100 101 110 111
bit label: ABC DEF GHI JKL MNO PQR STU VWX
</code></pre><p>would be encoded like this where spaces mark byte boundaries (3 bytes):</p><pre tabindex=0><code>bit value: 10001000 11000110 11111010
@@ -117,7 +120,9 @@ by keeping track of the number of values read.</p><p>Subtractions in steps 1) an
will the corresponding additions when decoding. Overflow should be allowed
and handled as wrapping around in 2’s complement notation so that the original
values are correctly restituted. This may require explicit care in some programming
-languages (for example by doing all arithmetic in the unsigned domain).</p><p>The following examples use 8 as the block size to keep the examples short,
+languages (for example by doing all arithmetic in the unsigned domain). Writers
+must not use more bits when bit packing the miniblock data than would be required
+to PLAIN encode the physical type (e.g. INT32 data must not use more than 32 bits).</p><p>The following examples use 8 as the block size to keep the examples short,
but in real cases it would be invalid.</p><h4 id=example-1>Example 1</h4><p>1, 2, 3, 4, 5</p><p>After step 1), we compute the deltas as:</p><p>1, 1, 1, 1</p><p>The minimum delta is 1 and after step 2, the relative deltas become:</p><p>0, 0, 0, 0</p><p>The final encoded data is:</p><p>header:
8 (block size), 1 (miniblock count), 5 (value count), 1 (first value)</p><p>block:
1 (minimum delta), 0 (bitwidth), (no data needed for bitwidth 0)</p><h4 id=example-2>Example 2</h4><p>7, 5, 3, 1, 2, 3, 4, 5, the deltas would be</p><p>-2, -2, -2, 1, 1, 1, 1</p><p>The minimum is -2, so the relative deltas are:</p><p>0, 0, 0, 3, 3, 3, 3</p><p>The encoded data is</p><p>header:
@@ -129,11 +134,10 @@ concatenated back to back. The expected savings is from the cost of encoding the
and possibly better compression in the data (it is no longer interleaved with the lengths).</p><p>The data stream looks like:</p><pre tabindex=0><code><Delta Encoded Lengths> <Byte Array Data>
</code></pre><p>For example, if the data was “Hello”, “World”, “Foobar”, “ABCDEF”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(5, 5, 6, 6) (the string lengths)</li><li>“HelloWorldFoobarABCDEF”</li></ul><h3 id=delta-strings-delta_byte_array--7>Delta Strings: (DELTA_BYTE_ARRAY = 7)</h3><p>Supported Types: BYTE_ARRAY, FIXED_LEN_BYTE_ARRAY</p><p>This is also known as incremental [...]
sequence of strings, store the prefix length of the previous entry plus the suffix.</p><p>For a longer description, see <a href=https://en.wikipedia.org/wiki/Incremental_encoding>https://en.wikipedia.org/wiki/Incremental_encoding</a>.</p><p>This is stored as a sequence of delta-encoded prefix lengths (DELTA_BINARY_PACKED), followed by
-the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
+the suffixes encoded as delta length byte arrays (DELTA_LENGTH_BYTE_ARRAY).</p><p>For example, if the data was “axis”, “axle”, “babble”, “babyhood”</p><p>then the encoded data would be comprised of the following segments:</p><ul><li>DeltaEncoding(0, 2, 0, 3) (the prefix lengths)</li><li>DeltaEncoding(4, 2, 6, 5) (the suffix lengths)</li><li>“axislebabbleyhood”</li></ul><p>Note that, even for FIXED_LEN_BYTE_ARRAY, all lengths are [...]
compression ratio and speed when a compression algorithm is used afterwards.</p><p>This encoding creates K byte-streams of length N where K is the size in bytes of the data
-type and N is the number of elements in the data sequence. Specifically, K is 4 for FLOAT
-type and 8 for DOUBLE type.
-The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
+type and N is the number of elements in the data sequence. For example, K is 4 for FLOAT
+type and 8 for DOUBLE type.</p><p>The bytes of each value are scattered to the corresponding streams. The 0-th byte goes to the
0-th stream, the 1-st byte goes to the 1-st stream and so on.
The streams are concatenated in the following order: 0-th stream, 1-st stream, etc.
The total length of encoded streams is K * N bytes. Because it does not have any metadata
@@ -142,6 +146,6 @@ is allowed inside the data page.</p><p>Example:
Original data is three 32-bit floats and for simplicity we look at their raw representation.</p><pre tabindex=0><code> Element 0 Element 1 Element 2
Bytes AA BB CC DD 00 11 22 33 A3 B4 C5 D6
</code></pre><p>After applying the transformation, the data has the following representation:</p><pre tabindex=0><code>Bytes AA 00 A3 BB 11 B4 CC 22 C5 DD 33 D6
-</code></pre><div class=td-page-meta__lastmod>Last modified November 12, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/a506efc9d2d708e86709841311f3220d828fb4cf>Minor: Fix various typos on the site (#133) (a506efc)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-i [...]
+</code></pre><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-foote [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/encryption/index.html b/output/docs/file-format/data-pages/encryption/index.html
index fb71d34..9adcbd5 100644
--- a/output/docs/file-format/data-pages/encryption/index.html
+++ b/output/docs/file-format/data-pages/encryption/index.html
@@ -1,12 +1,12 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
-1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><meta property="og:url" content="/docs/file-format/data-pages/encryption/"><meta property="og:site_n [...]
-1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="a [...]
-1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><meta itemprop=dateModified content="2025-11-12T06:31:27-05:00"><meta itemprop=wordCount content="39 [...]
-1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac5 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><meta property="og:url" content="/docs/file-format/data-pages/encryption/"><meta property="og:site_n [...]
+1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="a [...]
+1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop=wordCount content="42 [...]
+1 Problem Statement Existing data protection solutions (such as flat encryption of files, in-storage encryption, or use of an encrypting storage client) can be applied to Parquet files, but have various security or performance issues. An encryption mechanism, integrated in the Parquet format, allows for an optimal combination of data security, processing speed and encryption granularity."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/encryption.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Parquet%20Modular%20Encryption" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#1-problem-statement>1 Problem Statement</a></li><li><a href=#2-goals>2 Goals</a></li><li><a href=#3-technical-approach>3 Technical Approach</a></li><li><a href=#4-encryption-algorithms-and-keys>4 Encryption Algorithms and Keys</a><ul><li><a href=#41-aes-modes-used-in-parquet>4.1 AES modes used in Parquet</a> [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/da [...]
mechanism that encrypts and authenticates the file data and metadata - while allowing
for a regular Parquet functionality (columnar projection, predicate pushdown, encoding
and compression).</p><h2 id=1-problem-statement>1 Problem Statement</h2><p>Existing data protection solutions (such as flat encryption of files, in-storage encryption,
@@ -61,7 +61,20 @@ The unique input to CTR encryption is called IV (“initialization vector&rd
parts: a nonce and an initial counter field.</p><p>Parquet encryption uses the RBG-based (random bit generator) nonce construction as defined in
the section 8.2.2 of the NIST SP 800-38D document. For each encrypted module, Parquet generates a
unique nonce with a length of 12 bytes (96 bits). Notice: the NIST
-specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
+specification uses a term “IV” for what is called “nonce” in the Parquet encryption design.</p><h4 id=414-invocation-limit>4.1.4 Invocation limit</h4><p>According to the section 8.3 of the NIST SP 800-38D document, <em>“The total number of invocations
+of the authenticated encryption function shall not exceed 2^32, including all IV lengths and
+all instances of the authenticated encryption function with the given key”</em>. This restriction is
+related to the “uniqueness requirement of IVs and keys” (section 8 in the NIST spec) - <em>“if even
+one IV is ever repeated, then the implementation may be vulnerable”</em>. <em>“Compliance with this
+requirement is crucial to the security of GCM”</em>.</p><p>The bulk of modules in a Parquet file are page headers and data pages. Therefore, one encryption
+key shall not not be used for more than 2^31 (~2 billion) pages. In Parquet files encrypted with
+multiple keys (footer and column keys), the constraint on the number of invocations is applied
+to each key separately.</p><p>When running in the context of a larger system, any particular Parquet writer implementation likely
+does not have sufficient context to enforce key invocation limits system-wide. Therefore,
+the higher level system itself must arrange to supply keys appropriately to the various writer instances.</p><p>Parquet writer implementations should have a local invocation counter for each encryption key. If the
+counter exceeds 2^32, the implementation should return an error and produce no more cipherblocks.
+While this does not enforce a system-wide limit, it helps in distributed systems that provide different
+keys to different nodes (or generate unique keys in each node).</p><h3 id=42-parquet-encryption-algorithms>4.2 Parquet encryption algorithms</h3><h4 id=421-aes_gcm_v1>4.2.1 AES_GCM_V1</h4><p>This Parquet algorithm encrypts all modules by the GCM cipher, without padding. The AES GCM cipher
must be implemented by a cryptographic provider according to the NIST SP 800-38D specification.</p><p>In Parquet, an input to the GCM cipher is an encryption key, a 12-byte nonce, a plaintext and an
AAD. The output is a ciphertext with the length equal to that of plaintext, and a 16-byte authentication
tag used to verify the ciphertext and AAD integrity.</p><h4 id=422-aes_gcm_ctr_v1>4.2.2 AES_GCM_CTR_V1</h4><p>In this Parquet algorithm, all modules except pages are encrypted with the GCM cipher, as described
@@ -108,7 +121,7 @@ data set (table). This string is optionally passed by a writer upon file creatio
the AAD prefix is stored in an <code>aad_prefix</code> field in the file, and is made available to the readers.
This field is not encrypted. If a user is concerned about keeping the file identity inside the file,
the writer code can explicitly request Parquet not to store the AAD prefix. Then the aad_prefix field
-will be empty; AAD prefixes must be fully managed by the caller code and supplied explicitly to Parquet
+will be empty; AAD prefixes must be fully managed by the caller code and supplied explictly to Parquet
readers for each file.</p><p>The protection against swapping full files is optional. It is not enabled by default because
it requires the writers to generate and pass an AAD prefix.</p><p>A reader of a file created with an AAD prefix, should be able to verify the prefix (file identity)
by comparing it with e.g. the target table name, using a convention accepted in the organization.
@@ -164,7 +177,10 @@ decrypt it.</p><p>The row group ordinal, required for AAD suffix calculation, is
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/** Row group ordinal in the file **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>7</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>i16</span> <span style=color:#000>ordinal</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
+</span></span></code></pre></div><p>The integrity of this field is protected by authenticated encryption of the footer (FileMetaData). Therefore,
+the reader implementations can use either a local row group counter (ordinal) or the <code>RowGroup.ordinal</code>
+field as an input to AAD suffix calculation. The latter option can be helpful when different reader
+threads process different row groups in the same parquet file.</p><p>A <code>crypto_metadata</code> field is set in each ColumnChunk in the encrypted columns. ColumnCryptoMetaData
is a union - the actual structure is chosen depending on whether the column is encrypted with the
footer encryption key, or with a column-specific key. For the latter, a key metadata can be specified.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span> <span style=color:#000>EncryptionWithFooterKey</span> <span style=color:#000;font-weight:700>{</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
@@ -238,7 +254,7 @@ bytes) that they can’t parse this file.</p><div class=highlight><pre tabindex=
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * and (possibly) columns **/</span>
</span></span><span style=display:flex><span> <span style=color:#0000cf;font-weight:700>2</span><span style=color:#ce5c00;font-weight:700>:</span> <span style=color:#000>optional</span> <span style=color:#000>binary</span> <span style=color:#000>key_metadata</span>
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>}</span>
-</span></span></code></pre></div><p><img src=/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
+</span></span></code></pre></div><p><img src=/parquet-format/doc/images/FileLayoutEncryptionEF.png alt="File Layout - Encrypted footer"></p><h3 id=55-plaintext-footer-mode>5.5 Plaintext footer mode</h3><p>This mode allows legacy Parquet versions (released before the encryption support) to access
unencrypted columns in encrypted files - at a price of leaving certain metadata fields
unprotected in these files.</p><p>The plaintext footer mode can be useful during a transitional period in organizations where
some frameworks can’t be upgraded to a new Parquet library for a while. Data writers will
@@ -261,7 +277,7 @@ structure with the
AES GCM algorithm - using a footer signing key, and an AAD constructed according to the instructions
of the section 4.4. Only the nonce and GCM tag are stored in the file – as a 28-byte
fixed-length array, written right after the footer itself. The ciphertext is not stored,
-because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</ [...]
+because it is not required for footer integrity verification by readers.</p><table><thead><tr><th>nonce (12 bytes)</th><th>tag (16 bytes)</th></tr></thead><tbody></tbody></table><p>The plaintext footer mode sets the following fields in the the FileMetaData structure:</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-c data-lang=c><span style=display:flex><span><span style=color:#204a87;font-weight:700>stru [...]
</span></span><span style=display:flex><span><span style=color:#000;font-weight:700>...</span>
</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic>/**
</span></span></span><span style=display:flex><span><span style=color:#8f5902;font-style:italic> * Encryption algorithm. This field is set only in encrypted files
@@ -282,13 +298,13 @@ that contains the combined length of the footer and its signature. A final magic
“PAR1”, is written at the end of the
file. The same magic string is written at the beginning of the file (offset 0). The magic bytes
for plaintext footer mode are ‘PAR1’ to allow legacy readers to read projections of the file
-that do not include encrypted columns.</p><p><img src=/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
+that do not include encrypted columns.</p><p><img src=/parquet-format/doc/images/FileLayoutEncryptionPF.png alt="File Layout - Encrypted footer"></p><h2 id=6-encryption-overhead>6. Encryption Overhead</h2><p>The size overhead of Parquet modular encryption is negligible, since most of the encryption
operations are performed on pages (the minimal unit of Parquet data storage and compression).
The overhead order of magnitude is adding 1 byte per each ~30,000 bytes of original
data - calculated by comparing the page encryption overhead (nonce + tag + length = 32 bytes)
to the default page size (1 MB). This is a rough estimation, and can change with the encryption
algorithm (no 16-byte tag in AES_GCM_CTR_V1) and with page configuration or data encoding/compression.</p><p>The throughput overhead of Parquet modular encryption depends on whether AES enciphering is
done in software or hardware. In both cases, performing encryption on full pages (~1MB buffers)
-instead of on much smaller individual data values causes AES to work at its maximal speed.</p><div class=td-page-meta__lastmod>Last modified November 12, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/a506efc9d2d708e86709841311f3220d828fb4cf>Minor: Fix various typos on the site (#133) (a506efc)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 [...]
+instead of on much smaller individual data values causes AES to work at its maximal speed.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__l [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/errorrecovery/index.html b/output/docs/file-format/data-pages/errorrecovery/index.html
index 63e3205..6fc7284 100644
--- a/output/docs/file-format/data-pages/errorrecovery/index.html
+++ b/output/docs/file-format/data-pages/errorrecovery/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/errorrecovery.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Error%20Recovery" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/data-pages/>Data Pages</a></li><li class="brea [...]
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/da [...]
that column chunk is lost (but column chunks for this column in other row groups are
okay). If a page header is corrupt, the remaining pages in that chunk are lost. If
the data within a page is corrupt, that page is lost. The file will be more
@@ -14,4 +14,4 @@ Each file metadata would be cumulative and include all the row groups written so
far. Combining this with the strategy used for rc or avro files using sync markers,
a reader could recover partially written files.</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-li [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/index.html b/output/docs/file-format/data-pages/index.html
index f124300..927ed9d 100644
--- a/output/docs/file-format/data-pages/index.html
+++ b/output/docs/file-format/data-pages/index.html
@@ -6,17 +6,17 @@ The encoded values for the data page is always required. The definition and repe
repetition levels data definition levels data encoded values The value of uncompressed_page_size specified in the header is for all the 3 pieces combined.
The encoded values for the data page is always required. The definition and repetition levels are optional, based on the schema definition. If the column is not nested (i.e. the path to the column has length 1), we do not encode the repetition levels (it would always have the value 1). For data that is required, the definition levels are skipped (if encoded, it will always have the value of the max definition level)."><meta itemprop=dateModified content="2024-01-14T20:32:15+08:00"><meta [...]
repetition levels data definition levels data encoded values The value of uncompressed_page_size specified in the header is for all the 3 pieces combined.
-The encoded values for the data page is always required. The definition and repetition levels are optional, based on the schema definition. If the column is not nested (i.e. the path to the column has length 1), we do not encode the repetition levels (it would always have the value 1). For data that is required, the definition levels are skipped (if encoded, it will always have the value of the max definition level)."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca [...]
+The encoded values for the data page is always required. The definition and repetition levels are optional, based on the schema definition. If the column is not nested (i.e. the path to the column has length 1), we do not encode the repetition levels (it would always have the value 1). For data that is required, the definition levels are skipped (if encoded, it will always have the value of the max definition level)."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48e [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Data%20Pages/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Data%20Pages?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+ [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Data%20Pages" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Data Pages</li></ol></nav><div class=td-conte [...]
+<a id=print href=/_print/docs/file-format/data-pages/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>D [...]
header. No padding is allowed in the data page.
In order we have:</p><ol><li>repetition levels data</li><li>definition levels data</li><li>encoded values</li></ol><p>The value of <code>uncompressed_page_size</code> specified in the header is for all the 3 pieces combined.</p><p>The encoded values for the data page is always required. The definition and repetition levels
are optional, based on the schema definition. If the column is not nested (i.e.
the path to the column has length 1), we do not encode the repetition levels (it would
always have the value 1). For data that is required, the definition levels are
skipped (if encoded, it will always have the value of the max definition level).</p><p>For example, in the case where the column is non-nested and required, the data in the
-page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/data-pages/compression/>Compression</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/data-pages/encodings/>Encodings</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/data-pages/encryption/>Parqu [...]
+page is only the encoded values.</p><p>The supported encodings are described in Encodings.md</p><p>The supported compression codecs are described in Compression.md</p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/data-pages/compression/></a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/data-pages/encodings/></a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/data-pages/encryption/></a></h5><p></p></div><di [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/data-pages/index.xml b/output/docs/file-format/data-pages/index.xml
index 7db45c2..0ffe855 100644
--- a/output/docs/file-format/data-pages/index.xml
+++ b/output/docs/file-format/data-pages/index.xml
@@ -1,20 +1,24 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Data Pages on Parquet</title><link>/docs/file-format/data-pages/</link><description>Recent content in Data Pages on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/docs/file-format/data-pages/index.xml" rel="self" type="application/rss+xml"/><item><title>Compression</title><link>/docs/file-format/data-pages/compression [...]
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Data Pages on Parquet</title><link>/docs/file-format/data-pages/</link><description>Recent content in Data Pages on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/docs/file-format/data-pages/index.xml" rel="self" type="application/rss+xml"/><item><title/><link>/docs/file-format/data-pages/compression/</link><pubDate>M [...]
+<p>This document contains the specification of all supported compression codecs.</p>
+<h2 id="overview">Overview</h2>
<p>Parquet allows the data block inside dictionary pages and data pages to
be compressed for better space efficiency. The Parquet format supports
-several compression covering different areas in the compression ratio /
-processing cost spectrum.</p>
+several compression codecs covering different areas in the compression
+ratio / processing cost spectrum.</p>
<p>The detailed specifications of compression codecs are maintained externally
by their respective authors or maintainers, which we reference hereafter.</p>
<p>For all compression codecs except the deprecated <code>LZ4</code> codec, the raw data
of a (data or dictionary) page is fed <em>as-is</em> to the underlying compression
library, without any additional framing or padding. The information required
for precise allocation of compressed and decompressed buffers is written
-in the <code>PageHeader</code> struct.</p></description></item><item><title>Encodings</title><link>/docs/file-format/data-pages/encodings/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encodings/</guid><description><p><a name="PLAIN"></a></p>
+in the <code>PageHeader</code> struct.</p></description></item><item><title/><link>/docs/file-format/data-pages/encodings/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encodings/</guid><description><h1 id="parquet-encoding-definitions">Parquet encoding definitions</h1>
+<p>This file contains the specification of all supported encodings.</p>
+<p><a name="PLAIN"></a></p>
<h3 id="plain-plain--0">Plain: (PLAIN = 0)</h3>
<p>Supported Types: all</p>
<p>This is the plain encoding that must be supported for types. It is
intended to be the simplest encoding. Values are encoded back to back.</p>
-<p>The plain encoding is used whenever a more efficient encoding cannot be used. It
+<p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p>
<ul>
<li>BOOLEAN: <a href="/docs/file-format/data-pages/encodings/#BITPACKED">Bit Packed</a>, LSB first</li>
@@ -27,7 +31,8 @@ stores the data in the following format:</p>
<li>FIXED_LEN_BYTE_ARRAY: the bytes contained in the array</li>
</ul>
<p>For native types, this outputs the data as little endian. Floating
-point types are encoded in IEEE.</p></description></item><item><title>Parquet Modular Encryption</title><link>/docs/file-format/data-pages/encryption/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encryption/</guid><description><p>Parquet files containing sensitive information can be protected by the modular encryption
+point types are encoded in IEEE.</p></description></item><item><title/><link>/docs/file-format/data-pages/encryption/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encryption/</guid><description><h1 id="parquet-modular-encryption">Parquet Modular Encryption</h1>
+<p>Parquet files containing sensitive information can be protected by the modular encryption
mechanism that encrypts and authenticates the file data and metadata - while allowing
for a regular Parquet functionality (columnar projection, predicate pushdown, encoding
and compression).</p>
diff --git a/output/docs/file-format/extensibility/index.html b/output/docs/file-format/extensibility/index.html
index 55072f7..0e0c608 100644
--- a/output/docs/file-format/extensibility/index.html
+++ b/output/docs/file-format/extensibility/index.html
@@ -2,10 +2,10 @@
File Version: The file metadata contains a version. Encodings: Encodings are specified by enum and more can be added in the future. Page types: Additional page types can be added and safely skipped."><meta property="og:url" content="/docs/file-format/extensibility/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Extensibility"><meta property="og:description" content="There are many places in the format for compatible extensions:
File Version: The file metadata contains a version. Encodings: Encodings are specified by enum and more can be added in the future. Page types: Additional page types can be added and safely skipped."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2024-01-14T20:32:15+08:00"><meta itemprop=name content="Extensibility"><meta itemprop=description content="There [...]
File Version: The file metadata contains a version. Encodings: Encodings are specified by enum and more can be added in the future. Page types: Additional page types can be added and safely skipped."><meta itemprop=dateModified content="2024-01-14T20:32:15+08:00"><meta itemprop=wordCount content="43"><meta name=twitter:card content="summary"><meta name=twitter:title content="Extensibility"><meta name=twitter:description content="There are many places in the format for compatible extensions:
-File Version: The file metadata contains a version. Encodings: Encodings are specified by enum and more can be added in the future. Page types: Additional page types can be added and safely skipped."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c [...]
+File Version: The file metadata contains a version. Encodings: Encodings are specified by enum and more can be added in the future. Page types: Additional page types can be added and safely skipped."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/extensibility.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Extensibility" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Extensibility</li></ol></nav><div class=td-content><h1>E [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Extensibilit [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/implementationstatus/index.html b/output/docs/file-format/implementationstatus/index.html
index 9796c14..dda5ccf 100644
--- a/output/docs/file-format/implementationstatus/index.html
+++ b/output/docs/file-format/implementationstatus/index.html
@@ -14,12 +14,13 @@ arrow (C++) parquet-java (Java) arrow-go (Go) arrow-rs (Rust) cudf (cuDF C++) hy
Note: If you find out of date information, please help us improve the accuracy of this page by opening an issue or submitting a pull request.
Legend The value in each box means:
✅: supported ❌: not supported (R/W): partial reader/writer only support (blank): no data Implementations:
-arrow (C++) parquet-java (Java) arrow-go (Go) arrow-rs (Rust) cudf (cuDF C++) hyparquet (JavaScript) duckdb (C++) Physical types Physical types are defined by the enum Type in parquet.thrift"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel [...]
+arrow (C++) parquet-java (Java) arrow-go (Go) arrow-rs (Rust) cudf (cuDF C++) hyparquet (JavaScript) duckdb (C++) Physical types Physical types are defined by the enum Type in parquet.thrift"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/implementationstatus.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Implementation%20status" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#legend>Legend</a></li><li><a href=#physical-types>Physical types</a></li><li><a href=#logical-types>Logical types</a></li><li><a href=#encodings>Encodings</a></li><li><a href=#compressions>Compressions</a></li><li><a href=#other-format-level-features>Other format level features</a></li><li><a href=#high-level-d [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#legend>Legend</a></li><li><a href=#physical-types>Physical types</a></li><li><a href=#logical-types>Logical types</a></li><li><a href=#encodings>Encodings</a></li><li><a href=#compressions>Compressions</a></li><li><a href=#other-format-level-features>Other format level features</a></li><li><a href=#high-level-data-apis-for-parquet-feature-usage>High level data APIs for Parque [...]
implementations.</p><p><em>Note</em>: If you find out of date information, please help us improve the accuracy
of this page by opening an issue or submitting a pull request.</p><h3 id=legend>Legend</h3><p>The value in each box means:</p><ul><li>✅: supported</li><li>❌: not supported</li><li>(R/W): partial reader/writer only support</li><li>(blank): no data</li></ul><p>Implementations:</p><ul><li><a href=https://github.com/apache/arrow/tree/main/cpp/src/parquet>arrow</a> (C++)</li><li><a href=https://github.com/apache/parquet-java>parquet-java</a> (Java)</li><li><a href=https://github.com/apache/ar [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/index.html b/output/docs/file-format/index.html
index c277dea..60d89e5 100644
--- a/output/docs/file-format/index.html
+++ b/output/docs/file-format/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/file-format/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon- [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/file-format/><link rel=alternate type=application/rss+xml href=/docs/file-format/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon- [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=File%20Format" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>File Format</li></ol></nav><div class=td-content><h1>File Format</h1><div class=lead>Documentation about the Parquet File Format. [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>File Format</li></ol></nav><div class=td-content><h1>File Format</h1><div class=lead> [...]
<Column 1 Chunk 1>
<Column 2 Chunk 1>
...
@@ -25,6 +25,6 @@ start locations. More details on what is contained in the metadata can be found
in the Thrift definition.</p><p>File metadata is written after the data to allow for single pass writing.</p><p>Readers are expected to first read the file metadata to find all the column
chunks they are interested in. The columns chunks should then be read sequentially.</p><p>The format is explicitly designed to separate the metadata from the data. This
allows splitting columns into multiple files, as well as having a single metadata
-file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/configurations/>Configurations</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/extensibility/>Extensibility</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/metadata/>Metadata</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/types/>Types</a>< [...]
+file reference multiple parquet files.</p><p><img src=/images/FileLayout.gif alt="File Layout"></p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/binaryprotocolextensions/></a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/configurations/>Configurations</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/extensibility/>Extensibility</a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/metadata/ [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/index.xml b/output/docs/file-format/index.xml
index ab1df54..ff5560b 100644
--- a/output/docs/file-format/index.xml
+++ b/output/docs/file-format/index.xml
@@ -1,4 +1,12 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>File Format on Parquet</title><link>/docs/file-format/</link><description>Recent content in File Format on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/docs/file-format/index.xml" rel="self" type="application/rss+xml"/><item><title>Configurations</title><link>/docs/file-format/configurations/</link><pubDate>Mon, 01 [...]
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>File Format on Parquet</title><link>/docs/file-format/</link><description>Recent content in File Format on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/docs/file-format/index.xml" rel="self" type="application/rss+xml"/><item><title/><link>/docs/file-format/binaryprotocolextensions/</link><pubDate>Mon, 01 Jan 0001 00 [...]
+<p>The extension mechanism of the <code>binary</code> Thrift field-id <code>32767</code> has some desirable properties:</p>
+<ul>
+<li>Existing readers will ignore these extensions without any modifications</li>
+<li>Existing readers will ignore the extension bytes with little processing overhead</li>
+<li>The content of the extension is freeform and can be encoded in any format. This format is not restricted to Thrift.</li>
+<li>Extensions can be appended to existing Thrift serialized structs <a href="/docs/file-format/binaryprotocolextensions/#appending-extensions-to-thrift">without requiring Thrift libraries</a> for manipulation (or changes to the thrift IDL).</li>
+</ul>
+<p>Because only one field-id is reserved the extension bytes themselves require disambiguation; otherwise readers will not be able to decode extensions safely. This is left to implementers which MUST put enough unique state in their extension bytes for disambiguation. This can be relatively easily achieved by adding a <a href="https://en.wikipedia.org/wiki/Universally%5C_unique%5C_identifier">UUID</a> at the start or end of the extension bytes. The extension does not sp [...]
<p>Larger row groups allow for larger column chunks which makes it
possible to do larger sequential IO. Larger groups also require more buffering in
the write path (or a two pass write). We recommend large row groups (512MB - 1GB).
@@ -27,7 +35,8 @@ path for the column are defined. Repetition levels specify at what repeated fiel
in the path has the value repeated. The max definition and repetition levels can
be computed from the schema (i.e. how much nesting there is). This defines the
maximum number of bits required to store the levels (levels are defined for all
-values in the column).</p></description></item><item><title>Bloom Filter</title><link>/docs/file-format/bloomfilter/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/bloomfilter/</guid><description><h3 id="problem-statement">Problem statement</h3>
+values in the column).</p></description></item><item><title/><link>/docs/file-format/bloomfilter/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/bloomfilter/</guid><description><h1 id="parquet-bloom-filter">Parquet Bloom Filter</h1>
+<h3 id="problem-statement">Problem statement</h3>
<p>In their current format, column statistics and dictionaries can be used for predicate
pushdown. Statistics include minimum and maximum value, which can be used to filter out
values not in the range. Dictionaries are more specific, and readers can filter out values
@@ -37,9 +46,12 @@ space they occupy. This leaves columns with large cardinalities and widely separ
and max without support for predicate pushdown.</p></description></item><item><title>Nulls</title><link>/docs/file-format/nulls/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/nulls/</guid><description><p>Nullity is encoded in the definition levels (which is run-length encoded). NULL values
are not encoded in the data. For example, in a non-nested schema, a column with 1000 NULLs
would be encoded with run-length encoding (0, 1000 times) for the definition levels and
-nothing else.</p></description></item><item><title>Page Index</title><link>/docs/file-format/pageindex/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/pageindex/</guid><description><p>This document describes the format for column index pages in the Parquet
-footer. These pages contain statistics for DataPages and can be used to skip
-pages when scanning data in ordered and unordered columns.</p>
+nothing else.</p></description></item><item><title/><link>/docs/file-format/pageindex/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/pageindex/</guid><description><h1 id="parquet-page-index-layout-to-support-page-skipping">Parquet page index: Layout to Support Page Skipping</h1>
+<p>In Parquet, a <em>page index</em> is optional metadata for a
+ColumnChunk, containing statistics for DataPages that can be used
+to skip those pages when scanning in ordered and unordered columns.
+The page index is stored using the OffsetIndex and ColumnIndex structures,
+defined in <a href="src/main/thrift/parquet.thrift"><code>parquet.thrift</code></a></p>
<h2 id="problem-statement">Problem Statement</h2>
<p>In previous versions of the format, Statistics are stored for ColumnChunks in
ColumnMetaData and for individual pages inside DataPageHeader structs. When
diff --git a/output/docs/file-format/metadata/index.html b/output/docs/file-format/metadata/index.html
index c844087..6f5209c 100644
--- a/output/docs/file-format/metadata/index.html
+++ b/output/docs/file-format/metadata/index.html
@@ -10,15 +10,16 @@ File metadata In the diagram below, file metadata is described by the FileMetaDa
Page header Page header metadata (PageHeader and children in the diagram) is stored in-line with the page data, and is used in the reading and decoding of data."><meta itemprop=dateModified content="2025-03-05T23:19:07-08:00"><meta itemprop=wordCount content="89"><meta name=twitter:card content="summary"><meta name=twitter:title content="Metadata"><meta name=twitter:description content="There are two types of metadata: file metadata, and page header metadata.
All thrift structures are serialized using the TCompactProtocol. The full definition of these structures is given in the Parquet Thrift definition.
File metadata In the diagram below, file metadata is described by the FileMetaData structure. This file metadata provides offset and size information useful when navigating the Parquet file.
-Page header Page header metadata (PageHeader and children in the diagram) is stored in-line with the page data, and is used in the reading and decoding of data."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css rel=stylesheet integrity="sha256- [...]
+Page header Page header metadata (PageHeader and children in the diagram) is stored in-line with the page data, and is used in the reading and decoding of data."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256- [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/metadata.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Metadata" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#file-metadata>File metadata</a></li><li><a href=#page-header>Page header</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><a href=#file-metadata>File metadata</a></li><li><a href=#page-header>Page header</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcr [...]
definition of these structures is given in the Parquet
<a href=https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift>Thrift definition</a>.</p><h2 id=file-metadata>File metadata</h2><p>In the diagram below, file metadata is described by the <code>FileMetaData</code>
structure. This file metadata provides offset and size information useful
when navigating the Parquet file.</p><p><img src=/images/FileMetaData.svg alt="Parquet Metadata format"></p><h2 id=page-header>Page header</h2><p>Page header metadata (<code>PageHeader</code> and children in the diagram) is stored
in-line with the page data, and is used in the reading and decoding of data.</p><p><img src=/images/PageHeader.svg alt="Parquet PageHeader format"></p><div class=td-page-meta__lastmod>Last modified March 5, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/5ab1cc62cee7214f3c1f17e6b8a3a22e721eb6de>Update Metadata Diagrams (#106) (5ab1cc6)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/nestedencoding/index.html b/output/docs/file-format/nestedencoding/index.html
index b706356..7650252 100644
--- a/output/docs/file-format/nestedencoding/index.html
+++ b/output/docs/file-format/nestedencoding/index.html
@@ -1,8 +1,8 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/nestedencoding.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Nested%20Encoding" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Nested Encoding</li></ol></nav><div class=td-content><h1 [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Nested Encod [...]
repetition levels. Definition levels specify how many optional fields in the
path for the column are defined. Repetition levels specify at what repeated field
in the path has the value repeated. The max definition and repetition levels can
@@ -10,4 +10,4 @@ be computed from the schema (i.e. how much nesting there is). This defines the
maximum number of bits required to store the levels (levels are defined for all
values in the column).</p><p>Two encodings for the levels are supported BIT_PACKED and RLE. Only RLE is now used as it supersedes BIT_PACKED.</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/nulls/index.html b/output/docs/file-format/nulls/index.html
index e67e9ac..e651549 100644
--- a/output/docs/file-format/nulls/index.html
+++ b/output/docs/file-format/nulls/index.html
@@ -1,10 +1,10 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/nulls.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Nulls" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Nulls</li></ol></nav><div class=td-content><h1>Nulls</h1 [...]
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Nulls</li></ [...]
are not encoded in the data. For example, in a non-nested schema, a column with 1000 NULLs
would be encoded with run-length encoding (0, 1000 times) for the definition levels and
nothing else.</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/pageindex/index.html b/output/docs/file-format/pageindex/index.html
index b0fb69f..02e0bbb 100644
--- a/output/docs/file-format/pageindex/index.html
+++ b/output/docs/file-format/pageindex/index.html
@@ -1,14 +1,16 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
-Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><meta property="og:url" content="/docs/file-format/pageindex/"><meta property= [...]
-Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><meta property="og:locale" content="en"><meta property="og:type" content="arti [...]
-Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><meta itemprop=dateModified content="2024-01-14T20:32:15+08:00"><meta itemprop [...]
-Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><meta property="og:url" content="/docs/file-format/pageindex/"><meta property= [...]
+Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><meta property="og:locale" content="en"><meta property="og:type" content="arti [...]
+Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop [...]
+Problem Statement In previous versions of the format, Statistics are stored for ColumnChunks in ColumnMetaData and for individual pages inside DataPageHeader structs. When reading pages, a reader had to process the page header to determine whether the page could be skipped based on the statistics. This means the reader had to access all pages in a column, thus likely reading most of the column data from disk."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271d [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/pageindex.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Page%20Index" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><a href=#problem-statement>Problem Statement</a></li><li><a href=#goals>Goals</a></li><li><a href=#non-goals>Non-Goals</a></li><li><a href=#technical-approach>Technical Approach</a></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=bre [...]
-footer. These pages contain statistics for DataPages and can be used to skip
-pages when scanning data in ordered and unordered columns.</p><h2 id=problem-statement>Problem Statement</h2><p>In previous versions of the format, Statistics are stored for ColumnChunks in
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Page Index</ [...]
+ColumnChunk, containing statistics for DataPages that can be used
+to skip those pages when scanning in ordered and unordered columns.
+The page index is stored using the OffsetIndex and ColumnIndex structures,
+defined in <a href=src/main/thrift/parquet.thrift><code>parquet.thrift</code></a></p><h2 id=problem-statement>Problem Statement</h2><p>In previous versions of the format, Statistics are stored for ColumnChunks in
ColumnMetaData and for individual pages inside DataPageHeader structs. When
reading pages, a reader had to process the page header to determine
whether the page could be skipped based on the statistics. This means the reader
@@ -30,7 +32,7 @@ column are skipped, the corresponding rows in the other columns have to be
skipped. Hence the OffsetIndexes for each column in a RowGroup are stored
together.</li></ul><p>The new index structures are stored separately from RowGroup, near the footer.<br>This is done so that a reader does not have to pay the I/O and deserialization
cost for reading them if it is not doing selective scans. The index structures'
-location and length are stored in ColumnChunk.</p><p><img src=/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
+location and length are stored in ColumnChunk.</p><p><img src=/parquet-format/doc/images/PageIndexLayout.png alt="Page Index Layout"></p><p>Some observations:</p><ul><li>We don’t need to record the lower bound for the first page and the upper
bound for the last page, because the row group Statistics can provide that.
We still include those for the sake of uniformity, and the overhead should be
negligible.</li><li>We store lower and upper bounds for the values of each page. These may be the
@@ -48,6 +50,6 @@ reader can find matching pages by sequentially reading <code>min_values</code> a
indices, and page offsets to scan in each column. The reader can then
initialize a scanner for each column and fast forward them to the start row of
the scan.</p><p>The <code>min_values</code> and <code>max_values</code> are calculated based on the <code>column_orders</code>
-field in the <code>FileMetaData</code> struct of the footer.</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-foo [...]
+field in the <code>FileMetaData</code> struct of the footer.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1" [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/pageindex/src/main/thrift/parquet.thrift b/output/docs/file-format/pageindex/src/main/thrift/parquet.thrift
new file mode 100644
index 0000000..105ff6f
--- /dev/null
+++ b/output/docs/file-format/pageindex/src/main/thrift/parquet.thrift
@@ -0,0 +1,16 @@
+<!DOCTYPE html>
+<!-- Local redirect to latest parquet-thrift file -->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <title>Redirecting to parquet.thrift on GitHub</title>
+ <meta http-equiv="refresh" content="0; url=https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift">
+ <link rel="canonical" href="https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift">
+</head>
+<body>
+ <p>Redirecting to <a href="https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift">parquet.thrift on GitHub</a>...</p>
+ <script>
+ window.location.href = "https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift";
+ </script>
+</body>
+</html>
diff --git a/output/docs/file-format/nestedencoding/index.html b/output/docs/file-format/types/geospatial/index.html
similarity index 91%
copy from output/docs/file-format/nestedencoding/index.html
copy to output/docs/file-format/types/geospatial/index.html
index b706356..e0e8900 100644
--- a/output/docs/file-format/nestedencoding/index.html
+++ b/output/docs/file-format/types/geospatial/index.html
@@ -1,13 +1,65 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
-<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/nestedencoding.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
-<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+th [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Nested%20Encoding" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Nested Encoding</li></ol></nav><div class=td-content><h1 [...]
-repetition levels. Definition levels specify how many optional fields in the
-path for the column are defined. Repetition levels specify at what repeated field
-in the path has the value repeated. The max definition and repetition levels can
-be computed from the schema (i.e. how much nesting there is). This defines the
-maximum number of bits required to store the levels (levels are defined for all
-values in the column).</p><p>Two encodings for the levels are supported BIT_PACKED and RLE. Only RLE is now used as it supersedes BIT_PACKED.</p><div class=td-page-meta__lastmod>Last modified January 14, 2024: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/7cf58a9ec47d96608dfec9771179691301ede3ce>Sync site with format release v2.10.0 (7cf58a9)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Background The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM, XYZM) are defined by OpenGIS Implementation Specification for Geographic information - Simple feature access - Part 1: Common architecture, from OGC(Open Geospatial Consortium).
+The version of the OGC standard first used here is 1.2.1, but future versions may also be used if the WKB representation remains wire-compatible."><meta property="og:url" content="/docs/file-format/types/geospatial/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Parquet"><meta property="og:description" content="Geospatial Definitions This document contains the specification of geospatial types and statistics.
+Background The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM, XYZM) are defined by OpenGIS Implementation Specification for Geographic information - Simple feature access - Part 1: Common architecture, from OGC(Open Geospatial Consortium).
+The version of the OGC standard first used here is 1.2.1, but future versions may also be used if the WKB representation remains wire-compatible."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2025-12-09T11:38:22-08:00"><meta itemprop=name content="Parquet"><meta itemprop=description content="Geospatial Definitions This document contains the specification o [...]
+Background The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM, XYZM) are defined by OpenGIS Implementation Specification for Geographic information - Simple feature access - Part 1: Common architecture, from OGC(Open Geospatial Consortium).
+The version of the OGC standard first used here is 1.2.1, but future versions may also be used if the WKB representation remains wire-compatible."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop=wordCount content="931"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet"><meta name=twitter:description content="Geospatial Definitions This document contains the specification of geospatial types and statistics.
+Background The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM, XYZM) are defined by OpenGIS Implementation Specification for Geographic information - Simple feature access - Part 1: Common architecture, from OGC(Open Geospatial Consortium).
+The version of the OGC standard first used here is 1.2.1, but future versions may also be used if the WKB representation remains wire-compatible."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz6ZyqyY+C8epr [...]
+<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Types/Geospatial.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
+<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Types?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/types/> [...]
+Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM,
+XYZM) are defined by <a href="https://portal.ogc.org/files/?artifact_id=25355">OpenGIS Implementation Specification for Geographic
+information - Simple feature access - Part 1: Common architecture</a>,
+from <a href=https://www.ogc.org/standard/sfa/>OGC(Open Geospatial Consortium)</a>.</p><p>The version of the OGC standard first used here is 1.2.1, but future versions
+may also be used if the WKB representation remains wire-compatible.</p><h2 id=coordinate-reference-system>Coordinate Reference System</h2><p>Coordinate Reference System (CRS) is a mapping of how coordinates refer to
+locations on Earth.</p><p>The default CRS <code>OGC:CRS84</code> means that the geospatial features must be stored
+in the order of longitude/latitude based on the WGS84 datum.</p><p>Custom CRS can be specified by a string value. It is recommended to use an
+identifier-based approach like <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>.</p><p>For geographic CRS, longitudes are bound by [-180, 180] and latitudes are bound
+by [-90, 90].</p><h2 id=edge-interpolation-algorithm>Edge Interpolation Algorithm</h2><p>An algorithm for interpolating edges, and is one of the following values:</p><ul><li><code>spherical</code>: edges are interpolated as geodesics on a sphere.</li><li><code>vincenty</code>: <a href=https://en.wikipedia.org/wiki/Vincenty%27s_formulae>https://en.wikipedia.org/wiki/Vincenty%27s_formulae</a></li><li><code>thomas</code>: Thomas, Paul D. Spheroidal geodesics, reference systems, & local geom [...]
+logical types to store statistics of a column chunk. It is an optional field in
+the <code>ColumnMetaData</code> and contains <a href=/docs/file-format/types/geospatial/#bounding-box>Bounding Box</a> and <a href=/docs/file-format/types/geospatial/#geospatial-types>Geospatial
+Types</a> that are described below in detail.</p><h2 id=bounding-box>Bounding Box</h2><p>A geospatial instance has at least two coordinate dimensions: X and Y for 2D
+coordinates of each point. Please note that X is longitude/easting and Y is
+latitude/northing. A geospatial instance can optionally have Z and/or M values
+associated with each point.</p><p>The Z values introduce the third dimension coordinate. Usually they are used to
+indicate the height, or elevation.</p><p>M values are an opportunity for a geospatial instance to track a value in a
+fourth dimension. These values can be used as a linear reference value (e.g.,
+highway milepost value), a timestamp, or some other value as defined by the CRS.</p><p>Bounding box is defined as the thrift struct below in the representation of
+min/max value pair of coordinates from each axis. Note that X and Y Values are
+always present. Z and M are omitted for 2D geospatial instances.</p><p>When calculating a bounding box, null or NaN values in a coordinate
+dimension are skipped. For example, <code>POINT (1 NaN)</code> contributes a value to X
+but no values to Y, Z, or M dimension of the bounding box. If a dimension has
+only null or NaN values, that dimension is omitted from the bounding box. If
+either the X or Y dimension is missing, then the bounding box itself is not
+produced.</p><p>For the X values only, xmin may be greater than xmax. In this case, an object
+in this bounding box may match if it contains an X such that <code>x >= xmin</code> OR
+<code>x <= xmax</code>. This wraparound occurs only when the corresponding bounding box
+crosses the antimeridian line. In geographic terminology, the concepts of <code>xmin</code>,
+<code>xmax</code>, <code>ymin</code>, and <code>ymax</code> are also known as <code>westernmost</code>, <code>easternmost</code>,
+<code>southernmost</code> and <code>northernmost</code>, respectively.</p><p>For <code>GEOGRAPHY</code> types, X and Y values are restricted to the canonical ranges of
+[-180, 180] for X and [-90, 90] for Y.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-size:4><code class=language-thrift data-lang=thrift><span style=display:flex><span><span style=color:#204a87;font-weight:700>struct</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000>BoundingBox</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#000;font-weight:700>{</span><s [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>1</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>2</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>3</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>4</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>required</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>5</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>6</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>7</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#0000cf;font-weight:700>8</span><span style=color:#000;font-weight:700>:</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>optional</span><span style=color:#f8f8f8;text-decoration:underline> </span><span style=color:#204a87;font-weight:700>double</span><span style=color:#f8f8f8;text-decoration:underlin [...]
+</span></span></span><span style=display:flex><span><span style=color:#f8f8f8;text-decoration:underline></span><span style=color:#000;font-weight:700>}</span><span style=color:#f8f8f8;text-decoration:underline>
+</span></span></span></code></pre></div><h2 id=geospatial-types>Geospatial Types</h2><p>A list of geospatial types from all instances in the <code>GEOMETRY</code> or <code>GEOGRAPHY</code>
+column, or an empty list if they are not known.</p><p>This is borrowed from <a href="https://github.com/opengeospatial/geoparquet/blob/v1.1.0/format-specs/geoparquet.md?plain=1#L159">geometry_types of GeoParquet</a> except that
+values in the list are <a href=https://en.wikipedia.org/wiki/Well-known_text_representation_of_geometry#Well-known_binary>WKB (ISO-variant) integer codes</a>.
+Table below shows the most common geospatial types and their codes:</p><table><thead><tr><th style=text-align:left>Type</th><th style=text-align:left>XY</th><th style=text-align:left>XYZ</th><th style=text-align:left>XYM</th><th style=text-align:center>XYZM</th></tr></thead><tbody><tr><td style=text-align:left>Point</td><td style=text-align:left>0001</td><td style=text-align:left>1001</td><td style=text-align:left>2001</td><td style=text-align:center>3001</td></tr><tr><td style=text-alig [...]
+responsible for serializing and deserializing the CRS, respectively.</p><p>As a convention to maximize the interoperability, custom CRS values can be
+specified by a string of the format <code>type:identifier</code>, where <code>type</code> is one of
+the following values:</p><ul><li><code>srid</code>: <a href=https://en.wikipedia.org/wiki/Spatial_reference_system#Identifier>Spatial reference identifier</a>, <code>identifier</code> is the SRID itself.</li><li><code>projjson</code>: <a href=https://proj.org/en/stable/specifications/projjson.html>PROJJSON</a>, <code>identifier</code> is the name of a table property or a file property where the projjson string is stored.</li></ul><h1 id=coordinate-axis-order>Coordinate axis order</h1><p> [...]
+follows the de facto standard for axis order in WKB and is therefore always
+(x, y) where x is easting or longitude and y is northing or latitude. This
+ordering explicitly overrides the axis order as specified in the CRS.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 or [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/types/index.html b/output/docs/file-format/types/index.html
index 6a03c52..efe6387 100644
--- a/output/docs/file-format/types/index.html
+++ b/output/docs/file-format/types/index.html
@@ -2,11 +2,11 @@
- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><meta property="og:url" content="/docs/file-format/types/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Types"><met [...]
- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><meta property="og:locale" content="en"><meta property="og:type" content="website"><meta itemprop=name content="Types"><meta itemprop=description content [...]
- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><meta itemprop=dateModified content="2025-05-20T11:17:08+03:00"><meta itemprop=wordCount content="119"><meta name=twitter:card content="summary"><meta na [...]
-- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ [...]
+- BOOLEAN: 1 bit boolean - INT32: 32 bit signed ints - INT64: 64 bit signed ints - INT96: 96 bit signed ints (deprecated; only used by legacy implementations) - FLOAT: IEEE 32-bit floating point values - DOUBLE: IEEE 64-bit floating point values - BYTE_ARRAY: arbitrarily long byte arrays - FIXED_LEN_BYTE_ARRAY: fixed length byte arrays"><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/ [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Types/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Types?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Types" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Types</li></ol></nav><div class=td-content><h1>Typ [...]
+<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class="breadcrumb-item active" aria-current=page>Types< [...]
with a focus on how the types effect on disk storage. For example, 16-bit ints
are not explicitly supported in the storage format since they are covered by
32-bit ints with an efficient encoding. This reduces the complexity of implementing
@@ -18,6 +18,6 @@ readers and writers for the format. The types are:</p><pre tabindex=0><code> -
- DOUBLE: IEEE 64-bit floating point values
- BYTE_ARRAY: arbitrarily long byte arrays
- FIXED_LEN_BYTE_ARRAY: fixed length byte arrays
-</code></pre><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/types/logicaltypes/>Logical Types</a></h5><p></p></div></div><div class=td-page-meta__lastmod>Last modified May 20, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/fec6dbf4f693457112defa67851c52ce32fa91ac>Update _index.md (#114) (fec6dbf)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class= [...]
+</code></pre><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/file-format/types/geospatial/></a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/types/logicaltypes/></a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/types/variantshredding/></a></h5><p></p></div><div class=entry><h5><a href=/docs/file-format/types/variantencoding/></a></h5><p></p></div></div><div class=td-page-meta__lastmod>Last modified May 20, 2025: <a da [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/types/index.xml b/output/docs/file-format/types/index.xml
index 87a7709..9a039ec 100644
--- a/output/docs/file-format/types/index.xml
+++ b/output/docs/file-format/types/index.xml
@@ -1,7 +1,33 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Types on Parquet</title><link>/docs/file-format/types/</link><description>Recent content in Types on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/docs/file-format/types/index.xml" rel="self" type="application/rss+xml"/><item><title>Logical Types</title><link>/docs/file-format/types/logicaltypes/</link><pubDate>Mon, [...]
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Types on Parquet</title><link>/docs/file-format/types/</link><description>Recent content in Types on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/docs/file-format/types/index.xml" rel="self" type="application/rss+xml"/><item><title/><link>/docs/file-format/types/geospatial/</link><pubDate>Mon, 01 Jan 0001 00:00:00 + [...]
+<p>This document contains the specification of geospatial types and statistics.</p>
+<h1 id="background">Background</h1>
+<p>The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and
+Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM,
+XYZM) are defined by <a href="https://portal.ogc.org/files/?artifact_id=25355">OpenGIS Implementation Specification for Geographic
+information - Simple feature access - Part 1: Common architecture</a>,
+from <a href="https://www.ogc.org/standard/sfa/">OGC(Open Geospatial Consortium)</a>.</p>
+<p>The version of the OGC standard first used here is 1.2.1, but future versions
+may also be used if the WKB representation remains wire-compatible.</p></description></item><item><title/><link>/docs/file-format/types/logicaltypes/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/logicaltypes/</guid><description><h1 id="parquet-logical-type-definitions">Parquet Logical Type Definitions</h1>
+<p>Logical types are used to extend the types that parquet can be used to store,
by specifying how the primitive types should be interpreted. This keeps the set
of primitive types to a minimum and reuses parquet&rsquo;s efficient encodings. For
-example, strings are stored as byte arrays (binary) with a UTF8 annotation.
-These annotations define how to further decode and interpret the data.
-Annotations are stored as <code>LogicalType</code> fields in the file metadata and are
-documented in <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md">LogicalTypes.md</a></p></description></item></channel></rss>
\ No newline at end of file
+example, strings are stored with the primitive type <code>BYTE_ARRAY</code> with a <code>STRING</code>
+annotation.</p>
+<p>This file contains the specification for all logical types.</p>
+<h3 id="metadata">Metadata</h3>
+<p>The parquet format&rsquo;s <code>LogicalType</code> stores the type annotation. The annotation
+may require additional metadata fields, as well as rules for those fields.</p></description></item><item><title/><link>/docs/file-format/types/variantshredding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/variantshredding/</guid><description><h1 id="variant-shredding">Variant Shredding</h1>
+<p>The Variant type is designed to store and process semi-structured data efficiently, even with heterogeneous values.
+Query engines encode each Variant value in a self-describing format, and store it as a group containing <code>value</code> and <code>metadata</code> binary fields in Parquet.
+Since data is often partially homogeneous, it can be beneficial to extract certain fields into separate Parquet columns to further improve performance.
+This process is called <strong>shredding</strong>.</p>
+<p>Shredding enables the use of Parquet&rsquo;s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections.</p></description></item><item><title/><link>/docs/file-format/types/variantencoding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/variantencoding/</guid><description><h1 id="variant-binary-encoding">Variant Binary Encoding</h1>
+<p>A Variant represents a type that contains one of:</p>
+<ul>
+<li>Primitive: A type and corresponding value (e.g. INT, STRING)</li>
+<li>Array: An ordered list of Variant values</li>
+<li>Object: An unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate keys.</li>
+</ul>
+<p>A Variant is encoded with 2 binary values, the <a href="/docs/file-format/types/variantencoding/#value-encoding">value</a> and the <a href="/docs/file-format/types/variantencoding/#metadata-encoding">metadata</a>.</p>
+<p>There are a fixed number of allowed primitive types, provided in the table below.
+These represent a commonly supported subset of the <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md">logical types</a> allowed by the Parquet format.</p></description></item></channel></rss>
\ No newline at end of file
diff --git a/output/docs/file-format/types/logicaltypes/index.html b/output/docs/file-format/types/logicaltypes/index.html
index 2dd0bdd..fb94a38 100644
--- a/output/docs/file-format/types/logicaltypes/index.html
+++ b/output/docs/file-format/types/logicaltypes/index.html
@@ -1,13 +1,394 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+This file contains the specification for all logical types.
+Metadata The parquet format’s LogicalType stores the type annotation. The annotation may require additional metadata fields, as well as rules for those fields."><meta property="og:url" content="/docs/file-format/types/logicaltypes/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Parquet"><meta property="og:description" content="Parquet Logical Type Definitions Logical types are used to extend the types that parquet can be used to store, by specifying h [...]
+This file contains the specification for all logical types.
+Metadata The parquet format’s LogicalType stores the type annotation. The annotation may require additional metadata fields, as well as rules for those fields."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2025-12-09T11:38:22-08:00"><meta itemprop=name content="Parquet"><meta itemprop=description content="Parquet Logical Type Definitions Logical types are [...]
+This file contains the specification for all logical types.
+Metadata The parquet format’s LogicalType stores the type annotation. The annotation may require additional metadata fields, as well as rules for those fields."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop=wordCount content="4854"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet"><meta name=twitter:description content="Parquet Logical Type Definitions Logical types are used to extend the types that parquet can be used t [...]
+This file contains the specification for all logical types.
+Metadata The parquet format’s LogicalType stores the type annotation. The annotation may require additional metadata fields, as well as rules for those fields."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-R [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Types/logicaltypes.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Types?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section [...]
-<a href="https://github.com/apache/parquet-site/issues/new?title=Logical%20Types" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/types/>Types</a></li><li class="breadcrumb-item act [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/types/> [...]
by specifying how the primitive types should be interpreted. This keeps the set
of primitive types to a minimum and reuses parquet’s efficient encodings. For
-example, strings are stored as byte arrays (binary) with a UTF8 annotation.
-These annotations define how to further decode and interpret the data.
-Annotations are stored as <code>LogicalType</code> fields in the file metadata and are
-documented in <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>LogicalTypes.md</a></p><div class=td-page-meta__lastmod>Last modified October 23, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b35d71d57173edb593c0e620e132734f732b7044>Update logicaltypes.md (#38) (b35d71d)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left co [...]
+example, strings are stored with the primitive type <code>BYTE_ARRAY</code> with a <code>STRING</code>
+annotation.</p><p>This file contains the specification for all logical types.</p><h3 id=metadata>Metadata</h3><p>The parquet format’s <code>LogicalType</code> stores the type annotation. The annotation
+may require additional metadata fields, as well as rules for those fields.</p><p>There is an older representation of the logical type annotations called <code>ConvertedType</code>.
+To support backward compatibility with old files, readers should interpret <code>LogicalTypes</code>
+in the same way as <code>ConvertedType</code>, and writers should populate <code>ConvertedType</code> in the metadata
+according to well defined conversion rules.</p><h3 id=compatibility>Compatibility</h3><p>The Thrift definition of the metadata has two fields for logical types: <code>ConvertedType</code> and <code>LogicalType</code>.
+<code>ConvertedType</code> is an enum of all available annotations. Since Thrift enums can’t have additional type parameters,
+it is cumbersome to define additional type parameters, like decimal scale and precision
+(which are additional 32 bit integer fields on SchemaElement, and are relevant only for decimals) or time unit
+and UTC adjustment flag for Timestamp types. To overcome this problem, a new logical type representation was introduced into
+the metadata to replace <code>ConvertedType</code>: <code>LogicalType</code>. The new representation is a union of structs of logical types,
+this way allowing more flexible API, logical types can have type parameters.</p><p><code>ConvertedType</code> is deprecated. However, to maintain compatibility with old writers,
+Parquet readers should be able to read and interpret <code>ConvertedType</code> annotations
+in case <code>LogicalType</code> annotations are not present. Parquet writers must always write
+<code>LogicalType</code> annotations where applicable, but must also write the corresponding
+<code>ConvertedType</code> annotations (if any) to maintain compatibility with old readers.</p><p>Compatibility considerations are mentioned for each annotation in the corresponding section.</p><h2 id=string-types>String Types</h2><h3 id=string>STRING</h3><p><code>STRING</code> may only be used to annotate the <code>BYTE_ARRAY</code> primitive type and indicates
+that the byte array should be interpreted as a UTF-8 encoded character string.</p><p>The sort order used for <code>STRING</code> strings is unsigned byte-wise comparison.</p><p><em>Compatibility</em></p><p><code>STRING</code> corresponds to <code>UTF8</code> ConvertedType.</p><h3 id=enum>ENUM</h3><p><code>ENUM</code> annotates the <code>BYTE_ARRAY</code> primitive type and indicates that the value
+was converted from an enumerated type in another data model (e.g. Thrift, Avro, Protobuf).
+Applications using a data model lacking a native enum type should interpret <code>ENUM</code>
+annotated field as a UTF-8 encoded string.</p><p>The sort order used for <code>ENUM</code> values is unsigned byte-wise comparison.</p><h3 id=uuid>UUID</h3><p><code>UUID</code> annotates a 16-byte <code>FIXED_LEN_BYTE_ARRAY</code> primitive type. The value is
+encoded using big-endian, so that <code>00112233-4455-6677-8899-aabbccddeeff</code> is encoded
+as the bytes <code>00 11 22 33 44 55 66 77 88 99 aa bb cc dd ee ff</code>
+(This example is from <a href=https://en.wikipedia.org/wiki/Universally_unique_identifier>wikipedia’s UUID page</a>).</p><p>The sort order used for <code>UUID</code> values is unsigned byte-wise comparison.</p><h2 id=numeric-types>Numeric Types</h2><h3 id=signed-integers>Signed Integers</h3><p><code>INT</code> annotation can be used to specify the maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+For signed integers, the second parameter should be <code>true</code>,
+for example, a signed integer with bit width of 8 is defined as <code>INT(8, true)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, true)</code>, <code>INT(16, true)</code>, and <code>INT(32, true)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, true)</code> must annotate an <code>int64</code> primitive type. <code>INT(32, true)</code> and <code>INT(64, true)</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p>The sort order used for signed integer types is signed.</p><h3 id=unsigned-integers>Unsigned Integers</h3><p><code>INT</code> annotation can be used to specify unsigned integer types,
+along with a maximum number of bits in the stored value.
+The annotation has two parameters: bit width and sign.
+Allowed bit width values are <code>8</code>, <code>16</code>, <code>32</code>, <code>64</code>, and sign can be <code>true</code> or <code>false</code>.
+In case of unsigned integers, the second parameter should be <code>false</code>,
+for example, an unsigned integer with bit width of 8 is defined as <code>INT(8, false)</code>
+Implementations may use these annotations to produce smaller
+in-memory representations when reading data.</p><p>If a stored value is larger than the maximum allowed by the annotation, the
+behavior is not defined and can be determined by the implementation.
+Implementations must not write values that are larger than the annotation
+allows.</p><p><code>INT(8, false)</code>, <code>INT(16, false)</code>, and <code>INT(32, false)</code> must annotate an <code>int32</code> primitive type and
+<code>INT(64, false)</code> must annotate an <code>int64</code> primitive type.</p><p>The sort order used for unsigned integer types is unsigned.</p><h3 id=deprecated-integer-convertedtype>Deprecated integer ConvertedType</h3><p><code>INT_8</code>, <code>INT_16</code>, <code>INT_32</code>, and <code>INT_64</code> annotations can be also used to specify
+signed integers with 8, 16, 32, or 64 bit width.</p><p><code>INT_8</code>, <code>INT_16</code>, and <code>INT_32</code> must annotate an <code>int32</code> primitive type and
+<code>INT_64</code> must annotate an <code>int64</code> primitive type. <code>INT_32</code> and <code>INT_64</code> are
+implied by the <code>int32</code> and <code>int64</code> primitive types if no other annotation is
+present and should be considered optional.</p><p><code>UINT_8</code>, <code>UINT_16</code>, <code>UINT_32</code>, and <code>UINT_64</code> annotations can be also used to specify
+unsigned integers with 8, 16, 32, or 64 bit width.</p><p><code>UINT_8</code>, <code>UINT_16</code>, and <code>UINT_32</code> must annotate an <code>int32</code> primitive type and
+<code>UINT_64</code> must annotate an <code>int64</code> primitive type.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>INT_8</td><td>IntType (bitWidth = 8, isSigned = true)</td></tr><tr><td>INT_16</td><td>IntType (bitWidth = 16, isSigned = true)</td></tr><tr><td>INT_32</td><td>IntType (bitWidth = 32, isSigned = true)</td></tr><tr><td>INT_64</td><td>IntType (bitWidth = 64, isSigned = true)</td></tr><tr><td [...]
+the form <code>unscaledValue * 10^(-scale)</code>.</p><p>The primitive type stores an unscaled integer value. For <code>BYTE_ARRAY</code> and
+<code>FIXED_LEN_BYTE_ARRAY</code>, the unscaled number must be encoded as two’s complement using
+big-endian byte order (the most significant byte is the zeroth element). The
+scale stores the number of digits of that value that are to the right of the
+decimal point, and the precision stores the maximum number of digits supported
+in the unscaled value.</p><p>If not specified, the scale is 0. Scale must be zero or a positive integer less
+than or equal to the precision. Precision is required and must be a non-zero positive
+integer. A precision too large for the underlying type (see below) is an error.</p><p><code>DECIMAL</code> can be used to annotate the following types:</p><ul><li><code>int32</code>: for 1 <= precision <= 9</li><li><code>int64</code>: for 1 <= precision <= 18; precision < 10 will produce a
+warning</li><li><code>fixed_len_byte_array</code>: precision is limited by the array size. Length <code>n</code>
+can store <= <code>floor(log_10(2^(8*n - 1) - 1))</code> base-10 digits</li><li><code>byte_array</code>: <code>precision</code> is not limited, but is required. The minimum number of
+bytes to store the unscaled value should be used.</li></ul><p>The sort order used for <code>DECIMAL</code> values is signed comparison of the represented
+value.</p><p>If the column uses <code>int32</code> or <code>int64</code> physical types, then signed comparison of
+the integer values produces the correct ordering. If the physical type is
+fixed, then the correct ordering can be produced by flipping the
+most-significant bit in the first byte and then using unsigned byte-wise
+comparison.</p><p><em>Compatibility</em></p><p>To support compatibility with older readers, implementations of parquet-format should
+write <code>DecimalType</code> precision and scale into the corresponding SchemaElement field in metadata.</p><h3 id=float16>FLOAT16</h3><p>The <code>FLOAT16</code> annotation represents half-precision floating-point numbers in the 2-byte IEEE little-endian format.</p><p>Used in contexts where precision is traded off for smaller footprint and potentially better performance.</p><p>The primitive type is a 2-byte <code>FIXED_LEN_BYTE_ARRAY</code>.</p><p>The sort order for <code>FLOAT16</cod [...]
+annotate an <code>int32</code> that stores the number of days from the Unix epoch, 1
+January 1970.</p><p>The sort order used for <code>DATE</code> is signed.</p><h3 id=time>TIME</h3><p><code>TIME</code> is used for a logical time type without a date with millisecond or microsecond precision.
+The type has two type parameters: UTC adjustment (<code>true</code> or <code>false</code>)
+and unit (<code>MILLIS</code> or <code>MICROS</code>, <code>NANOS</code>).</p><p><code>TIME</code> with unit <code>MILLIS</code> is used for millisecond precision.
+It must annotate an <code>int32</code> that stores the number of
+milliseconds after midnight.</p><p><code>TIME</code> with unit <code>MICROS</code> is used for microsecond precision.
+It must annotate an <code>int64</code> that stores the number of
+microseconds after midnight.</p><p><code>TIME</code> with unit <code>NANOS</code> is used for nanosecond precision.
+It must annotate an <code>int64</code> that stores the number of
+nanoseconds after midnight.</p><p>The sort order used for <code>TIME</code> is signed.</p><h4 id=deprecated-time-convertedtype>Deprecated time ConvertedType</h4><p><code>TIME_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int32</code>.</p><p><code>TIME_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIME</code> logical
+type that is UTC normalized and has <code>MICROS</code> precision. Like the logical type
+counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local time semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local time with legacy <code>TIME_MICROS</code> and <code>TIME_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local time with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIME_MILLIS</td><td>TimeType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIME_MICROS</td><td>TimeType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimeType</td><td rowspan=3>isAdj [...]
+<code>int64</code> number that can be decoded into year, month, day, hour, minute, second
+and subsecond fields using calculations detailed below. Please note that a value
+defined this way does not necessarily correspond to a single instant on the
+time-line and such interpretations are allowed on purpose.</p><p>The <code>TIMESTAMP</code> type has two type parameters:</p><ul><li><code>isAdjustedToUTC</code> must be either <code>true</code> or <code>false</code>.</li><li><code>unit</code> must be one of <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>. This list is subject
+to potential expansion in the future. Upon reading, unknown <code>unit</code>-s must
+be handled as unsupported features (rather than as errors in the data files).</li></ul><h4 id=instant-semantics-timestamps-normalized-to-utc>Instant semantics (timestamps normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=true</code> is defined as the number of
+milliseconds, microseconds or nanoseconds (depending on the <code>unit</code>
+parameter being <code>MILLIS</code>, <code>MICROS</code> or <code>NANOS</code>, respectively) elapsed since the
+Unix epoch, 1970-01-01 00:00:00 UTC. Each such value unambiguously identifies a
+single instant on the time-line.</p><p>For example, in a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>, the
+number 172800000 corresponds to 1970-01-03 00:00:00 UTC, because it is equal to
+2 * 24 * 60 * 60 * 1000, therefore it is exactly two days from the reference
+point, the Unix epoch. In Java, this calculation can be achieved by calling
+<code>Instant.ofEpochMilli(172800000)</code>.</p><p>As a slightly more complicated example, if one wants to store 1970-01-03
+00:00:00 (UTC+01:00) as a <code>TIMESTAMP(isAdjustedToUTC=true, unit=MILLIS)</code>,
+first the time zone offset has to be dealt with. By normalizing the timestamp to
+UTC, we calculate what time in UTC corresponds to the same instant: 1970-01-02
+23:00:00 UTC. This is 1 day and 23 hours after the epoch, therefore it can be
+encoded as the number (24 + 23) * 60 * 60 * 1000 = 169200000.</p><p>Please note that time zone information gets lost in this process. Upon reading a
+value back, we can only reconstruct the instant, but not the original
+representation. In practice, such timestamps are typically displayed to users in
+their local time zones, therefore they may be displayed differently depending on
+the execution environment.</p><h4 id=local-semantics-timestamps-not-normalized-to-utc>Local semantics (timestamps not normalized to UTC)</h4><p>A <code>TIMESTAMP</code> with <code>isAdjustedToUTC=false</code> represents year, month, day, hour,
+minute, second and subsecond fields in a local timezone, <em>regardless of what
+specific time zone is considered local</em>. This means that such timestamps should
+always be displayed the same way, regardless of the local time zone in effect.
+On the other hand, without additional information such as an offset or
+time-zone, these values do not identify instants on the time-line unambiguously,
+because the corresponding instants would depend on the local time zone.</p><p>Using a single number to represent a local timestamp is a lot less intuitive
+than for instants. One must use a local timestamp as the reference point and
+calculate the elapsed time between the actual timestamp and the reference point.
+The problem is that the result may depend on the local time zone, for example
+because there may have been a daylight saving time change between the two
+timestamps.</p><p>The solution to this problem is to use a simplification that makes the result
+easy to calculate and independent of the timezone. Treating every day as
+consisting of exactly 86400 seconds and ignoring DST changes altogether allows
+us to unambiguously represent a local timestamp as a difference from a reference
+local timestamp. We define the reference local timestamp to be 1970-01-01
+00:00:00 (note the lack of UTC at the end, as this is not an instant). This way
+the encoding of local timestamp values becomes very similar to the encoding of
+instant values. For example, in a <code>TIMESTAMP(isAdjustedToUTC=false, unit=MILLIS)</code>, the number 172800000 corresponds to 1970-01-03 00:00:00
+(note the lack of UTC at the end), because it is exactly two days from the
+reference point (172800000 = 2 * 24 * 60 * 60 * 1000).</p><p>Another way to get to the same definition is to treat the local timestamp value
+<em>as if</em> it were in UTC and store it as an instant. For example, if we treat the
+local timestamp 1970-01-03 00:00:00 <em>as if</em> it were the instant 1970-01-03
+00:00:00 UTC, we can store it as 172800000. When reading 172800000 back, we can
+reconstruct the instant 1970-01-03 00:00:00 UTC and convert it to a local
+timestamp <em>as if</em> we were in the UTC time zone, resulting in 1970-01-03
+00:00:00. In Java, this can be achieved by calling
+<code>LocalDateTime.ofEpochSecond(172800, 0, ZoneOffset.UTC)</code>.</p><p>Please note that while from a practical point of view this second definition is
+equivalent to the first one, from a theoretical point of view only the first
+definition can be considered correct, the second one just “incidentally” leads
+to the same results. Nevertheless, this second definition is worth mentioning as
+well, because it is relatively widespread and it can lead to confusion,
+especially due to its usage of UTC in the calculations. One can stumble upon
+code, comments and specifications ambiguously stating that a timestamp “is
+stored in UTC”. In some contexts, it means that it is <em>normalized</em> to UTC and
+acts as an instant. In some other contexts though, it means the exact opposite,
+namely that the timestamp is stored <em>as if</em> it were in UTC and acts as a
+local timestamp in reality.</p><h4 id=common-considerations>Common considerations</h4><p>Every possible <code>int64</code> number represents a valid timestamp, but depending on the
+precision, the corresponding year may be outside of the practical everyday
+limits and implementations may choose to only support a limited range.</p><p>On the other hand, not every combination of year, month, day, hour, minute,
+second and subsecond values can be encoded into an <code>int64</code>. Most notably:</p><ul><li>An arbitrary combination of timestamp fields can not be encoded as a single
+number if the values for some of the fields are outside of their normal range
+(where the “normal range” corresponds to everyday usage). For example, neither
+of the following can be represented in a timestamp:<ul><li>hour = -1</li><li>hour = 25</li><li>minute = 61</li><li>month = 13</li><li>day = 29, month = 2, year = any non-leap year</li></ul></li><li>Due to the range of the <code>int64</code> type, timestamps using the <code>NANOS</code> unit
+can only represent values between 1677-09-21 00:12:43 and 2262-04-11 23:47:16.
+Values outside of this range can not be represented with the <code>NANOS</code>
+unit. (Other precisions have similar limits but those are outside of the
+domain for practical everyday usage.)</li></ul><p>The sort order used for <code>TIMESTAMP</code> is signed.</p><h4 id=deprecated-timestamp-convertedtype>Deprecated timestamp ConvertedType</h4><p><code>TIMESTAMP_MILLIS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MILLIS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p><code>TIMESTAMP_MICROS</code> is the deprecated ConvertedType counterpart of a <code>TIMESTAMP</code>
+logical type that is UTC normalized and has <code>MICROS</code> precision. Like the logical
+type counterpart, it must annotate an <code>int64</code>.</p><p>Despite there is no exact corresponding ConvertedType for local timestamp semantic,
+in order to support forward compatibility with those libraries, which annotated
+their local timestamps with legacy <code>TIMESTAMP_MICROS</code> and <code>TIMESTAMP_MILLIS</code> annotation,
+Parquet writer implementation <em>must</em> annotate local timestamps with legacy annotations too,
+as shown below.</p><p><em>Backward compatibility:</em></p><table><thead><tr><th>ConvertedType</th><th>LogicalType</th></tr></thead><tbody><tr><td>TIMESTAMP_MILLIS</td><td>TimestampType (isAdjustedToUTC = true, unit = MILLIS)</td></tr><tr><td>TIMESTAMP_MICROS</td><td>TimestampType (isAdjustedToUTC = true, unit = MICROS)</td></tr></tbody></table><p><em>Forward compatibility:</em></p><table><tr colspan=3><th colspan=3>LogicalType</th><th>ConvertedType</th></tr><tr><td rowspan=6>TimestampTyp [...]
+<code>fixed_len_byte_array</code> of length 12. This array stores three little-endian
+unsigned integers that represent durations at different granularities of time.
+The first stores a number in months, the second stores a number in days, and
+the third stores a number in milliseconds. This representation is independent
+of any particular timezone or date.</p><p>Each component in this representation is independent of the others. For
+example, there is no requirement that a large number of days should be
+expressed as a mix of months and days because there is not a constant
+conversion from days to months.</p><p>The sort order used for <code>INTERVAL</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=embedded-types>Embedded Types</h2><p>Embedded types do not have type-specific orderings.</p><h3 id=json>JSON</h3><p><code>JSON</code> is used for an embedded JSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as a UTF-8 encoded character
+string of valid JSON as defined by the <a href=http://json.org/>JSON specification</a></p><p>The sort order used for <code>JSON</code> is unsigned byte-wise comparison.</p><h3 id=bson>BSON</h3><p><code>BSON</code> is used for an embedded BSON document. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. The <code>BYTE_ARRAY</code> data is interpreted as an encoded BSON document as
+defined by the <a href=http://bsonspec.org/spec.html>BSON specification</a>.</p><p>The sort order used for <code>BSON</code> is unsigned byte-wise comparison.</p><h3 id=variant>VARIANT</h3><p><code>VARIANT</code> is used for a Variant value. It must annotate a group. The group must
+contain a field named <code>metadata</code> and a field named <code>value</code>. Both fields must have
+type <code>binary</code>, which is also called <code>BYTE_ARRAY</code> in the Parquet thrift definition.
+The <code>VARIANT</code> annotated group can be used to store either an unshredded Variant
+value, or a shredded Variant value.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</code> must be of type <code>binary</code> (called <code>BYTE_ARRAY</code>
+in the Parquet thrift definition).</li><li>The <code>metadata</code> field is required and must be a valid Variant metadata component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>When present, the <code>value</code> field must be a valid Variant value component,
+as defined by the <a href=/docs/file-format/types/variantencoding/>Variant binary encoding specification</a>.</li><li>The <code>value</code> field is required for unshredded Variant values.</li><li>The <code>value</code> field is optional and may be null only when parts of the Variant
+value are shredded according to the <a href=/docs/file-format/types/variantshredding/>Variant shredding specification</a>.</li></ul><p>This is the expected representation of an unshredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_unshredded (VARIANT) {
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group variant_shredded (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><h3 id=geometry>GEOMETRY</h3><p><code>GEOMETRY</code> is used for geospatial features in the Well-Known Binary (WKB) format
+with linear/planar edges interpolation. It must annotate a <code>BYTE_ARRAY</code>
+primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has only one type parameter:</p><ul><li><code>crs</code>: An optional string value for CRS. If unset, the CRS defaults to
+<code>"OGC:CRS84"</code>, which means that the geometries must be stored in longitude,
+latitude based on the WGS84 datum.</li></ul><p>The sort order used for <code>GEOMETRY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h3 id=geography>GEOGRAPHY</h3><p><code>GEOGRAPHY</code> is used for geospatial features in the WKB format with an explicit
+(non-linear/non-planar) edges interpolation algorithm. It must annotate a
+<code>BYTE_ARRAY</code> primitive type. See <a href=/docs/file-format/types/geospatial/>Geospatial.md</a> for more detail.</p><p>The type has two type parameters:</p><ul><li><code>crs</code>: An optional string value for CRS. It must be a geographic CRS, where
+longitudes are bound by [-180, 180] and latitudes are bound by [-90, 90].
+If unset, the CRS defaults to <code>"OGC:CRS84"</code>.</li><li><code>algorithm</code>: An optional enum value to describes the edge interpolation
+algorithm. Supported values are: <code>SPHERICAL</code>, <code>VINCENTY</code>, <code>THOMAS</code>, <code>ANDOYER</code>,
+<code>KARNEY</code>. If unset, the algorithm defaults to <code>SPHERICAL</code>.</li></ul><p>The sort order used for <code>GEOGRAPHY</code> is undefined. When writing data, no min/max
+statistics should be saved for this type and if such non-compliant statistics
+are found during reading, they must be ignored.</p><h2 id=nested-types>Nested Types</h2><p>This section specifies how <code>LIST</code> and <code>MAP</code> can be used to encode nested types
+by adding group levels around repeated fields that are not present in the data.</p><p>This does not affect repeated fields that are not annotated: A repeated field
+that is neither contained by a <code>LIST</code>- or <code>MAP</code>-annotated group nor annotated
+by <code>LIST</code> or <code>MAP</code> should be interpreted as a required list of required
+elements where the element type is the type of the field.</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// List<Integer> (non-null list, non-null elements)
+repeated int32 num;
+
+// List<Tuple<Integer, String>> (non-null list, non-null elements)
+repeated group my_list {
+ required int32 num;
+ optional binary str (STRING);
+}
+</code></pre><p>For all fields in the schema, implementations should use either <code>LIST</code> and
+<code>MAP</code> annotations <em>or</em> unannotated repeated fields, but not both. When using
+the annotations, no unannotated repeated types are allowed.</p><h3 id=lists>Lists</h3><p><code>LIST</code> is used to annotate types that should be interpreted as lists.</p><p><code>LIST</code> must always annotate a 3-level structure:</p><pre tabindex=0><code><list-repetition> group <name> (LIST) {
+ repeated group list {
+ <element-repetition> <element-type> element;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>LIST</code> that contains a
+single field named <code>list</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the list is nullable.</li><li>The middle level, named <code>list</code>, must be a repeated group with a single
+field named <code>element</code>.</li><li>The <code>element</code> field encodes the list’s element type and repetition. Element
+repetition must be <code>required</code> or <code>optional</code>.</li></ul><p>The following examples demonstrate two of the possible lists of string values.</p><pre tabindex=0><code>// List<String> (list non-null, elements nullable)
+required group my_list (LIST) {
+ repeated group list {
+ optional binary element (STRING);
+ }
+}
+
+// List<String> (list nullable, elements non-null)
+optional group my_list (LIST) {
+ repeated group list {
+ required binary element (STRING);
+ }
+}
+</code></pre><p>Element types can be nested structures. For example, a list of lists:</p><pre tabindex=0><code>// List<List<Integer>>
+optional group array_of_arrays (LIST) {
+ repeated group list {
+ required group element (LIST) {
+ repeated group list {
+ required int32 element;
+ }
+ }
+ }
+}
+</code></pre><h4 id=backward-compatibility-rules>Backward-compatibility rules</h4><p>New writer implementations should always produce the 3-level LIST structure shown
+above. However, historically data files have been produced that use different
+structures to represent list-like data, and readers may include compatibility
+measures to interpret them as intended.</p><p>It is required that the repeated group of elements is named <code>list</code> and that
+its element field is named <code>element</code>. However, these names may not be used in
+existing data and should not be enforced as errors when reading. For example,
+the following field schema should produce a nullable list of non-null strings,
+even though the repeated group is named <code>element</code>.</p><pre tabindex=0><code>optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ };
+}
+</code></pre><p>Some existing data does not include the inner element layer, resulting in a
+<code>LIST</code> that annotates a 2-level structure. Unlike the 3-level structure, the
+repetition of a 2-level structure can be <code>optional</code>, <code>required</code>, or <code>repeated</code>.
+When it is <code>repeated</code>, the <code>LIST</code>-annotated 2-level structure can only serve as
+an element within another <code>LIST</code>-annotated 2-level structure.</p><p>For backward-compatibility, the type of elements in <code>LIST</code>-annotated structures
+should always be determined by the following rules:</p><ol><li>If the repeated field is not a group, then its type is the element type and
+elements are required.</li><li>If the repeated field is a group with multiple fields, then its type is the
+element type and elements are required.</li><li>If the repeated field is a group with one field with <code>repeated</code> repetition,
+then its type is the element type and elements are required.</li><li>If the repeated field is a group with one field and is named either <code>array</code>
+or uses the <code>LIST</code>-annotated group’s name with <code>_tuple</code> appended then the
+repeated type is the element type and elements are required.</li><li>Otherwise, the repeated field’s type is the element type with the repeated
+field’s repetition.</li></ol><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>WARNING: writers should not produce list types like these examples! They are
+just for the purpose of reading existing data for backward-compatibility.
+
+// Rule 1: List<Integer> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated int32 element;
+}
+
+// Rule 2: List<Tuple<String, Integer>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group element {
+ required binary str (STRING);
+ required int32 num;
+ };
+}
+
+// Rule 3: List<List<Integer>> (nullable outer list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array (LIST) {
+ repeated int32 array;
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group array {
+ required binary str (STRING);
+ };
+}
+
+// Rule 4: List<OneTuple<String>> (nullable list, non-null elements)
+optional group my_list (LIST) {
+ repeated group my_list_tuple {
+ required binary str (STRING);
+ };
+}
+
+// Rule 5: List<String> (nullable list, nullable elements)
+optional group my_list (LIST) {
+ repeated group element {
+ optional binary str (STRING);
+ };
+}
+</code></pre><h3 id=maps>Maps</h3><p><code>MAP</code> is used to annotate types that should be interpreted as a map from keys
+to values. <code>MAP</code> must annotate a 3-level structure:</p><pre tabindex=0><code><map-repetition> group <name> (MAP) {
+ repeated group key_value {
+ required <key-type> key;
+ <value-repetition> <value-type> value;
+ }
+}
+</code></pre><ul><li>The outer-most level must be a group annotated with <code>MAP</code> that contains a
+single field named <code>key_value</code>. The repetition of this level must be either
+<code>optional</code> or <code>required</code> and determines whether the map is nullable.</li><li>The middle level, named <code>key_value</code>, must be a repeated group with a <code>key</code>
+field for map keys and, optionally, a <code>value</code> field for map values. It must
+not contain any other values.</li><li>The <code>key</code> field encodes the map’s key type. This field must have
+repetition <code>required</code> and must always be present. It must always be the first
+field of the repeated <code>key_value</code> group.</li><li>The <code>value</code> field encodes the map’s value type and repetition. This field can
+be <code>required</code>, <code>optional</code>, or omitted. It must always be the second field of
+the repeated <code>key_value</code> group if present. In case of not present, it can be
+represented as a map with all null values or as a set of keys.</li></ul><p>The following example demonstrates the type for a non-null map from strings to
+nullable integers:</p><pre tabindex=0><code>// Map<String, Integer>
+required group my_map (MAP) {
+ repeated group key_value {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><p>If there are multiple key-value pairs for the same key, then the final value
+for that key must be the last value. Other values may be ignored or may be
+added with replacement to the map container in the order that they are encoded.
+The <code>MAP</code> annotation should not be used to encode multi-maps using duplicate
+keys.</p><h4 id=backward-compatibility-rules-1>Backward-compatibility rules</h4><p>It is required that the repeated group of key-value pairs is named <code>key_value</code>
+and that its fields are named <code>key</code> and <code>value</code>. However, these names may not
+be used in existing data and should not be enforced as errors when reading.
+(<code>key</code> and <code>value</code> can be identified by their position in case of misnaming.)</p><p>Some existing data incorrectly used <code>MAP_KEY_VALUE</code> in place of <code>MAP</code>. For
+backward-compatibility, a group annotated with <code>MAP_KEY_VALUE</code> that is not
+contained by a <code>MAP</code>-annotated group should be handled as a <code>MAP</code>-annotated
+group.</p><p>Examples that can be interpreted using these rules:</p><pre tabindex=0><code>// Map<String, Integer> (nullable map, non-null values)
+optional group my_map (MAP) {
+ repeated group map {
+ required binary str (STRING);
+ required int32 num;
+ }
+}
+
+// Map<String, Integer> (nullable map, nullable values)
+optional group my_map (MAP_KEY_VALUE) {
+ repeated group map {
+ required binary key (STRING);
+ optional int32 value;
+ }
+}
+</code></pre><h2 id=unknown-always-null>UNKNOWN (always null)</h2><p>Sometimes, when discovering the schema of existing data, values are always null
+and there’s no type information.
+The <code>UNKNOWN</code> type can be used to annotate a column that is always null.
+(Similar to Null type in Avro and Arrow)</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/file-format/types/variantencoding/index.html b/output/docs/file-format/types/variantencoding/index.html
new file mode 100644
index 0000000..fa3dc7f
--- /dev/null
+++ b/output/docs/file-format/types/variantencoding/index.html
@@ -0,0 +1,247 @@
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Primitive: A type and corresponding value (e.g. INT, STRING) Array: An ordered list of Variant values Object: An unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate keys. A Variant is encoded with 2 binary values, the value and the metadata.
+There are a fixed number of allowed primitive types, provided in the table below. These represent a commonly supported subset of the logical types allowed by the Parquet format."><meta property="og:url" content="/docs/file-format/types/variantencoding/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Parquet"><meta property="og:description" content="Variant Binary Encoding A Variant represents a type that contains one of:
+Primitive: A type and corresponding value (e.g. INT, STRING) Array: An ordered list of Variant values Object: An unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate keys. A Variant is encoded with 2 binary values, the value and the metadata.
+There are a fixed number of allowed primitive types, provided in the table below. These represent a commonly supported subset of the logical types allowed by the Parquet format."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2025-12-09T11:38:22-08:00"><meta itemprop=name content="Parquet"><meta itemprop=description content="Variant Binary Encoding A Variant [...]
+Primitive: A type and corresponding value (e.g. INT, STRING) Array: An ordered list of Variant values Object: An unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate keys. A Variant is encoded with 2 binary values, the value and the metadata.
+There are a fixed number of allowed primitive types, provided in the table below. These represent a commonly supported subset of the logical types allowed by the Parquet format."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop=wordCount content="3062"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet"><meta name=twitter:description content="Variant Binary Encoding A Variant represents a type that contains one of:
+Primitive: A type and corresponding value (e.g. INT, STRING) Array: An ordered list of Variant values Object: An unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate keys. A Variant is encoded with 2 binary values, the value and the metadata.
+There are a fixed number of allowed primitive types, provided in the table below. These represent a commonly supported subset of the logical types allowed by the Parquet format."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet i [...]
+<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Types/VariantEncoding.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
+<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Types?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/types/> [...]
+These represent a commonly supported subset of the <a href=https://github.com/apache/parquet-format/blob/master/LogicalTypes.md>logical types</a> allowed by the Parquet format.</p><p>The Variant Binary Encoding allows representation of semi-structured data (e.g. JSON) in a form that can be efficiently queried by path.
+The design is intended to allow efficient access to nested data even in the presence of very wide or deep structures.</p><p>Another motivation for the representation is that (aside from metadata) each nested Variant value is contiguous and self-contained.
+For example, in a Variant containing an Array of Variant values, the representation of an inner Variant value, when paired with the metadata of the full variant, is itself a valid Variant.</p><p>This document describes the Variant Binary Encoding scheme.
+Variant fields can also be <em>shredded</em>.
+Shredding refers to extracting some elements of the variant into separate columns for more efficient extraction/filter pushdown.
+The <a href=/docs/file-format/types/variantshredding/>Variant Shredding specification</a> describes the details of shredding Variant values as typed Parquet columns.</p><h2 id=variant-in-parquet>Variant in Parquet</h2><p>A Variant value in Parquet is represented by a group with 2 fields, named <code>value</code> and <code>metadata</code>.</p><ul><li>The Variant group must be annotated with the <code>VARIANT</code> logical type.</li><li>Both fields <code>value</code> and <code>metadata</c [...]
+ required binary metadata;
+ required binary value;
+}
+</code></pre><p>This is an example representation of a shredded Variant in Parquet:</p><pre tabindex=0><code>optional group shredded_variant_name (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The <code>VARIANT</code> annotation places no additional restrictions on the repetition of Variant groups, but repetition may be restricted by containing types (such as <code>MAP</code> and <code>LIST</code>).
+The Variant group name is the name of the Variant column.</p><h2 id=metadata-encoding>Metadata encoding</h2><p>The encoded metadata always starts with a header byte.</p><pre tabindex=0><code> 7 6 5 4 3 0
+ +-------+---+---+---------------+
+header | | | | version |
+ +-------+---+---+---------------+
+ ^ ^
+ | +-- sorted_strings
+ +-- offset_size_minus_one
+</code></pre><p>The <code>version</code> is a 4-bit value that must always contain the value <code>1</code>.
+<code>sorted_strings</code> is a 1-bit value indicating whether dictionary strings are sorted and unique.
+<code>offset_size_minus_one</code> is a 2-bit value providing the number of bytes per dictionary size and offset field.
+The actual number of bytes, <code>offset_size</code>, is <code>offset_size_minus_one + 1</code>.</p><p>The entire metadata is encoded as the following diagram shows:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+metadata | header |
+ +-----------------------+
+ | |
+ : dictionary_size : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : offset : <-- unsigned little-endian, `offset_size` bytes
+ | | (`dictionary_size + 1` offsets)
+ +-----------------------+
+ | |
+ : bytes :
+ | |
+ +-----------------------+
+</code></pre><p>The metadata is encoded first with the <code>header</code> byte, then <code>dictionary_size</code> which is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the number of string values in the dictionary.
+Next, is an <code>offset</code> list, which contains <code>dictionary_size + 1</code> values.
+Each <code>offset</code> is an unsigned little-endian value of <code>offset_size</code> bytes, and represents the starting byte offset of the i-th string in <code>bytes</code>.
+The first <code>offset</code> value will always be <code>0</code>, and the last <code>offset</code> value will always be the total length of <code>bytes</code>.
+The last part of the metadata is <code>bytes</code>, which stores all the string values in the dictionary.
+All string values must be UTF-8 encoded strings.</p><h3 id=metadata-encoding-grammar>Metadata encoding grammar</h3><p>The grammar for encoded metadata is as follows</p><pre tabindex=0><code>metadata: <header> <dictionary_size> <dictionary>
+header: 1 byte (<version> | <sorted_strings> << 4 | (<offset_size_minus_one> << 6))
+version: a 4-bit version ID. Currently, must always contain the value 1
+sorted_strings: a 1-bit value indicating whether metadata strings are sorted
+offset_size_minus_one: 2-bit value providing the number of bytes per dictionary size and offset field.
+dictionary_size: `offset_size` bytes. unsigned little-endian value indicating the number of strings in the dictionary
+dictionary: <offset>* <bytes>
+offset: `offset_size` bytes. unsigned little-endian value indicating the starting position of the ith string in `bytes`. The list should contain `dictionary_size + 1` values, where the last value is the total length of `bytes`.
+bytes: UTF-8 encoded dictionary string values
+</code></pre><p>Notes:</p><ul><li>Offsets are relative to the start of the <code>bytes</code> array.</li><li>The length of the ith string can be computed as <code>offset[i+1] - offset[i]</code>.</li><li>The offset of the first string is always equal to 0 and is therefore redundant. It is included in the spec to simplify in-memory-processing.</li><li><code>offset_size_minus_one</code> indicates the number of bytes per <code>dictionary_size</code> and <code>offset</code> entry. I.e. a valu [...]
+ +------------------------------------+------------+
+value | value_header | basic_type | <-- value_metadata
+ +------------------------------------+------------+
+ | |
+ : value_data : <-- 0 or more bytes
+ | |
+ +-------------------------------------------------+
+</code></pre><h3 id=basic-type>Basic Type</h3><p>The <code>basic_type</code> is 2-bit value that represents which basic type the Variant value is.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>basic types table</a> shows what each value represents.</p><h3 id=value-header>Value Header</h3><p>The <code>value_header</code> is a 6-bit value that contains more information about the type, and the format depends on the <code>basic_type</code>.</p><h4 id=value-header-for-primitive-type-basic_type0>Value Header for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>v [...]
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows what each value represents.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | primitive_header |
+ +-----------------------+
+</code></pre><h4 id=value-header-for-short-string-basic_type1>Value Header for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_header</code> is a 6-bit <code>short_string_header</code>.</p><pre tabindex=0><code> 5 0
+ +-----------------------+
+value_header | short_string_header |
+ +-----------------------+
+</code></pre><p>The <code>short_string_header</code> value is the length of the string.</p><h4 id=value-header-for-object-basic_type2>Value Header for Object (<code>basic_type</code>=2)</h4><p>When <code>basic_type</code> is <code>2</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, <code>field_id_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 4 3 2 1 0
+ +---+---+-------+-------+
+value_header | | | | |
+ +---+---+-------+-------+
+ ^ ^ ^
+ | | +-- field_offset_size_minus_one
+ | +-- field_id_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> and <code>field_id_size_minus_one</code> are 2-bit values that represent the number of bytes used to encode the field offsets and field ids.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code> and <code>field_id_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h4 id=value-header-for-array-basic_type3>Value Header for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_header</code> is made up of <code>field_offset_size_minus_one</code>, and <code>is_large</code>.</p><pre tabindex=0><code> 5 3 2 1 0
+ +-----------+---+-------+
+value_header | | | |
+ +-----------+---+-------+
+ ^ ^
+ | +-- field_offset_size_minus_one
+ +-- is_large
+</code></pre><p><code>field_offset_size_minus_one</code> is a 2-bit value that represents the number of bytes used to encode the field offset.
+The actual number of bytes is computed as <code>field_offset_size_minus_one + 1</code>.
+<code>is_large</code> is a 1-bit value that indicates how many bytes are used to encode the number of elements.
+If <code>is_large</code> is <code>0</code>, 1 byte is used, and if <code>is_large</code> is <code>1</code>, 4 bytes are used.</p><h3 id=value-data>Value Data</h3><p>The <code>value_data</code> encoding format depends on the type specified by <code>value_metadata</code>.
+For some types, the <code>value_data</code> will be 0-bytes.</p><h4 id=value-data-for-primitive-type-basic_type0>Value Data for Primitive type (<code>basic_type</code>=0)</h4><p>When <code>basic_type</code> is <code>0</code>, <code>value_data</code> depends on the <code>primitive_header</code> value.
+The <a href=/docs/file-format/types/variantencoding/#encoding-types>primitive types table</a> shows the encoding format for each primitive type.</p><h4 id=value-data-for-short-string-basic_type1>Value Data for Short string (<code>basic_type</code>=1)</h4><p>When <code>basic_type</code> is <code>1</code>, <code>value_data</code> is the sequence of UTF-8 encoded bytes that represents the string.</p><h4 id=value-data-for-object-basic_type2>Value Data for Object (<code>basic_type</code>=2)</ [...]
+The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+object value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_id : <-- unsigned little-endian, `field_id_size` bytes
+ | | (`num_elements` field_ids)
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An object <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the object.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a list of <code>field_id</code> values.
+There are <code>num_elements</code> number of entries and each <code>field_id</code> is an unsigned little-endian value of <code>field_id_size</code> bytes.
+A <code>field_id</code> is an index into the dictionary in the metadata.
+The <code>field_id</code> list is followed by a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the end of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th key-value pair of the object, the key is the metadata dictionary entry indexed by the i-th <code>field_id</code>, and the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><p>The field ids and field offsets must be in lexicographical order of the corresponding field names in the metadata dictionary.
+However, the actual <code>value</code> entries do not need to be in any particular order.
+This implies that the <code>field_offset</code> values may not be monotonically increasing.
+For example, for the following object:</p><pre tabindex=0><code>{
+ "c": 3,
+ "b": 2,
+ "a": 1
+}
+</code></pre><p>The <code>field_id</code> list must be <code>[<id for key "a">, <id for key "b">, <id for key "c">]</code>, in lexicographical order.
+The <code>field_offset</code> list must be <code>[<offset for value 1>, <offset for value 2>, <offset for value 3>, <last offset>]</code>.
+The <code>value</code> list can be in any order.</p><h4 id=value-data-for-array-basic_type3>Value Data for Array (<code>basic_type</code>=3)</h4><p>When <code>basic_type</code> is <code>3</code>, <code>value_data</code> encodes an array. The encoding format is shown in the following diagram:</p><pre tabindex=0><code> 7 0
+ +-----------------------+
+array value_data | |
+ : num_elements : <-- unsigned little-endian, 1 or 4 bytes
+ | |
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : field_offset : <-- unsigned little-endian, `field_offset_size` bytes
+ | | (`num_elements + 1` field_offsets)
+ +-----------------------+
+ | |
+ : value :
+ | |
+ +-----------------------+
+ :
+ +-----------------------+
+ | |
+ : value : <-- (`num_elements` values)
+ | |
+ +-----------------------+
+</code></pre><p>An array <code>value_data</code> begins with <code>num_elements</code>, a 1-byte or 4-byte unsigned little-endian value, representing the number of elements in the array.
+The size in bytes of <code>num_elements</code> is indicated by <code>is_large</code> in the <code>value_header</code>.
+Next, is a <code>field_offset</code> list.
+There are <code>num_elements + 1</code> number of entries and each <code>field_offset</code> is an unsigned little-endian value of <code>field_offset_size</code> bytes.
+A <code>field_offset</code> represents the byte offset (relative to the first byte of the first <code>value</code>) where the i-th <code>value</code> starts.
+The last <code>field_offset</code> points to the byte after the last byte of the last <code>value</code>.
+The <code>field_offset</code> list is followed by the <code>value</code> list.
+There are <code>num_elements</code> number of <code>value</code> entries and each <code>value</code> is an encoded Variant value.
+For the i-th array entry, the value is the Variant <code>value</code> starting from the i-th <code>field_offset</code> byte offset.</p><h3 id=value-encoding-grammar>Value encoding grammar</h3><p>The grammar for an encoded value is:</p><pre tabindex=0><code>value: <value_metadata> <value_data>?
+value_metadata: 1 byte (<basic_type> | (<value_header> << 2))
+basic_type: ID from Basic Type table. <value_header> must be a corresponding variation
+value_header: <primitive_header> | <short_string_header> | <object_header> | <array_header>
+primitive_header: ID from Primitive Type table. <val> must be a corresponding variation of <primitive_val>
+short_string_header: unsigned string length in bytes from 0 to 63
+object_header: (is_large << 4 | field_id_size_minus_one << 2 | field_offset_size_minus_one)
+array_header: (is_large << 2 | field_offset_size_minus_one)
+value_data: <primitive_val> | <short_string_val> | <object_val> | <array_val>
+primitive_val: see table for binary representation
+short_string_val: UTF-8 encoded bytes
+object_val: <num_elements> <field_id>* <field_offset>* <fields>
+array_val: <num_elements> <field_offset>* <fields>
+num_elements: a 1 or 4 byte unsigned little-endian value (depending on is_large in <object_header>/<array_header>)
+field_id: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_id_size_minus_one in <object_header>), indexing into the dictionary
+field_offset: a 1, 2, 3 or 4 byte unsigned little-endian value (depending on field_offset_size_minus_one in <object_header>/<array_header>), providing the offset in bytes within fields
+fields: <value>*
+</code></pre><p>Each <code>value_data</code> must correspond to the type defined by <code>value_metadata</code>. Boolean and null types do not have a corresponding <code>value_data</code>, since their type defines their value.</p><p>Each <code>array_val</code> and <code>object_val</code> must contain exactly <code>num_elements + 1</code> values for <code>field_offset</code>.
+The last entry is the offset that is one byte past the last field (i.e. the total size of all fields in bytes).
+All offsets are relative to the first byte of the first field in the object/array.</p><p><code>field_id_size_minus_one</code> and <code>field_offset_size_minus_one</code> indicate the number of bytes per field ID/offset.
+For example, a value of 0 indicates 1-byte IDs, 1 indicates 2-byte IDs, 2 indicates 3 byte IDs and 3 indicates 4-byte IDs.
+The <code>is_large</code> flag for arrays and objects is used to indicate whether the number of elements is indicated using a one or four byte value.
+When more than 255 elements are present, <code>is_large</code> must be set to true.
+It is valid for an implementation to use a larger value than necessary for any of these fields (e.g. <code>is_large</code> may be true for an object with less than 256 elements).</p><p>The “short string” basic type may be used as an optimization to fold string length into the type byte for strings less than 64 bytes.
+It is semantically identical to the “string” primitive type.</p><p>The Decimal type contains a scale, but no precision. The implied precision of a decimal value is <code>floor(log_10(val)) + 1</code>.</p><h2 id=encoding-types>Encoding types</h2><p><em>Variant basic types</em></p><table><thead><tr><th>Basic Type</th><th>ID</th><th>Description</th></tr></thead><tbody><tr><td>Primitive</td><td><code>0</code></td><td>One of the primitive types</td></tr><tr><td>Short string</td><t [...]
+For example, a user expression operating on a string value containing “hello” should behave the same, whether it is encoded with the short string optimization, or long string encoding.
+Similarly, user expressions operating on an <em>int8</em> value of 1 should behave the same as a decimal16 with scale 2 and unscaled value 100.</p><p><em>Decimal table</em></p><table><thead><tr><th>Decimal Precision</th><th>Decimal value type</th><th>Variant Physical Type</th></tr></thead><tbody><tr><td>1 <= precision <= 9</td><td>int32</td><td>decimal4</td></tr><tr><td>10 <= precision <= 18</td><td>int64</td><td>decimal8</td></tr><tr><td>19 <= precision <= 38</td><td>i [...]
+This includes the dictionary key string values, the “short string” values, and the “long string” values.</p><h2 id=object-field-id-order-and-uniqueness>Object field ID order and uniqueness</h2><p>For objects, field IDs and offsets must be listed in the order of the corresponding field names, sorted lexicographically (using unsigned byte ordering for UTF-8).
+Note that the field values themselves are not required to follow this order.
+As a result, offsets will not necessarily be listed in ascending order.
+The field values are not required to be in the same order as the field IDs, to enable flexibility when constructing Variant values.</p><p>An implementation may rely on this field ID order in searching for field names.
+E.g. a binary search on field IDs (combined with metadata lookups) may be used to find a field with a given name.</p><p>Field names are case-sensitive.
+Field names are required to be unique for each object.
+It is an error for an object to contain two fields with the same name, whether or not they have distinct dictionary IDs.</p><h2 id=versions-and-extensions>Versions and extensions</h2><p>An implementation is not expected to parse a Variant value whose metadata version is higher than the version supported by the implementation.
+However, new types may be added to the specification without incrementing the version ID.
+In such a situation, an implementation should be able to read the rest of the Variant value if desired.</p><h2 id=shredding>Shredding</h2><p>A single Variant object may have poor read performance when only a small subset of fields are needed.
+A better approach is to create separate columns for individual fields, referred to as shredding or subcolumnarization.
+<a href=/docs/file-format/types/variantshredding/>VariantShredding.md</a> describes the Variant shredding specification in Parquet.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print-none"><div class=container-fluid><div cla [...]
+2025
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/community/index.html b/output/docs/file-format/types/variantshredding/index.html
similarity index 85%
copy from output/community/index.html
copy to output/docs/file-format/types/variantshredding/index.html
index 280bc6b..14b6333 100644
--- a/output/community/index.html
+++ b/output/docs/file-format/types/variantshredding/index.html
@@ -1,23 +1,165 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/community/><link rel=alternate type=application/rss+xml href=/community/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png si [...]
-<a href=https://www.apache.org>Apache Software Foundation</a>
-and is developed and maintained by a contributors from around the world.
-We'd love you to join us!</p></span></div></div></section><section class="row td-box td-box--white td-box--height-auto linkbox"><div class="col-xs-12 col-sm-6 col-md-6 col-lg-6"><h2>Learn and Connect</h2><p><b>Development Mailing List</b>
-Most Parquet related discussion occurs on the developer mailing list,
-<a href=mailto:dev@parquet.apache.org>dev@parquet.apache.org</a>:</p><ul><li title=Subscribe><a href=mailto:dev-subscribe@parquet.apache.org><i class="fa fa-envelope-circle-check"></i> Subscribe</a>:
-Send an email to subscribe to the list</li><li title=Post><a href=mailto:dev@parquet.apache.org><i class="fa fa-envelope"></i> Post</a>:
-Send a message to the list</li><li title=Archives><a href=https://lists.apache.org/list.html?dev@parquet.apache.org><i class="fa fa-box-archive"></i> Archives</a>:
-View archives of past discussions.</li></ul><p>We also hold bi-weekly video calls to coordinate efforts.
-Everybody is welcome to bring a topic or just listen in.
-We send meeting reminders with links to the
-developer mailing list before each meeting, and a summary after each meeting.
-See
-<a href=https://lists.apache.org/thread/bjdkscmx7zvgfbw0wlfttxy8h6v3f71t>here</a>
-for more details on how the meeting is run, the
-<a href="https://www.youtube.com/playlist?list=PLsc4WKPCP34Q6RFVeqEVXJshpAReTmgUj">archives</a>
-for recordings of past meetings, and
-<a href=https://docs.google.com/document/d/e/2PACX-1vSDHW7gvG8eO6aIxaIVPrZSqYYhtRDb5W1imnbpM4QRYNPsTwEO1fU5z7SEhVIFa4YqWJeSRJ9tcXYS/pub>here</a>
-for agendas of past meetings.</p><ul><li title=Subscribe><a href=https://groups.google.com/g/apache-parquet-community-sync/about><i class="fa fa-users"></i> Join Group</a>:
-Join the Google group to access the calendar invite, agenda, and recordings of past calls.</li></ul></div><div class="col-xs-12 col-sm-6 col-md-6 col-lg-6"><h2>Develop and Contribute</h2><p>If you want to get more involved by contributing to Parquet, please see our
-<a href=/docs/contribution-guidelines/>Developer Guide.</a></p></div></section></main><footer class="td-footer row d-print-none"><div class=container-fluid><div class="row mx-md-2"><div class="td-footer__left col-6 col-sm-4 order-sm-1"><ul class=td-footer__links-list><li class=td-footer__links-item data-bs-toggle=tooltip title="Post to Mailing List" aria-label="Post to Mailing List"><a target=_blank rel=noopener href=mailto:dev@parquet.apache.org aria-label="Post to Mailing List"><i clas [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections."><meta property="og:url" content="/docs/file-format/types/variantshredding/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Parquet"><meta property="og:description" content="Variant Shredding The Variant type is designed to store and process semi-structured data efficiently, even with heterogeneous [...]
+Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections."><meta property="og:locale" content="en"><meta property="og:type" content="article"><meta property="article:section" content="docs"><meta property="article:modified_time" content="2025-12-09T11:38:22-08:00"><meta itemprop=name content="Parquet"><meta itemprop=description content="Variant Shredding The Variant type is designed to stor [...]
+Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections."><meta itemprop=dateModified content="2025-12-09T11:38:22-08:00"><meta itemprop=wordCount content="2171"><meta name=twitter:card content="summary"><meta name=twitter:title content="Parquet"><meta name=twitter:description content="Variant Shredding The Variant type is designed to store and process semi-structured data efficiently, eve [...]
+Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css rel=stylesheet integrity="sha256-Rrz6 [...]
+<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/File%20Format/Types/VariantShredding.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
+<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/File%20Format/Types?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section [...]
+<a href="https://github.com/apache/parquet-site/issues/new?title=" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
+<a id=print href=/_print/docs/file-format/types/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/file-format/>File Format</a></li><li class=breadcrumb-item><a href=/docs/file-format/types/> [...]
+Query engines encode each Variant value in a self-describing format, and store it as a group containing <code>value</code> and <code>metadata</code> binary fields in Parquet.
+Since data is often partially homogeneous, it can be beneficial to extract certain fields into separate Parquet columns to further improve performance.
+This process is called <strong>shredding</strong>.</p><p>Shredding enables the use of Parquet’s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections.</p><p>For example, the query <code>SELECT variant_get(event, '$.event_ts', 'timestamp') FROM tbl</code> only needs to load field <code>event_ts</code>, and if that column is shredded, it can be read by columnar projection without reading or deserializing the rest of the < [...]
+Similarly, for the query <code>SELECT * FROM tbl WHERE variant_get(event, '$.event_type', 'string') = 'signup'</code>, the <code>event_type</code> shredded column metadata can be used for skipping and to lazily load the rest of the Variant.</p><h2 id=variant-metadata>Variant Metadata</h2><p>Variant metadata is stored in the top-level Variant group in a binary <code>metadata</code> column regardless of whether the Variant value is shredded.</p><p>All <code>value</code> columns within the [...]
+All field names of a Variant, whether shredded or not, must be present in the metadata.</p><h2 id=value-shredding>Value Shredding</h2><p>Variant values are stored in Parquet fields named <code>value</code>.
+Each <code>value</code> field may have an associated shredded field named <code>typed_value</code> that stores the value when it matches a specific type.
+When <code>typed_value</code> is present, readers <strong>must</strong> reconstruct shredded values according to this specification.</p><p>For example, a Variant field, <code>measurement</code> may be shredded as long values by adding <code>typed_value</code> with type <code>int64</code>:</p><pre tabindex=0><code>required group measurement (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional int64 typed_value;
+}
+</code></pre><p>The Parquet columns used to store variant metadata and values must be accessed by name, not by position.</p><p>The series of measurements <code>34, null, "n/a", 100</code> would be stored as:</p><table><thead><tr><th>Value</th><th><code>metadata</code></th><th><code>value</code></th><th><code>typed_value</code></th></tr></thead><tbody><tr><td>34</td><td><code>01 00</code> v1/empty</td><td>null</td><td><code>34</code></td></tr><tr><td>null</td><td><code>01 00</code> v1/emp [...]
+Values in the two fields must be interpreted according to the following table:</p><table><thead><tr><th><code>value</code></th><th><code>typed_value</code></th><th>Meaning</th></tr></thead><tbody><tr><td>null</td><td>null</td><td>The value is missing; only valid for shredded object fields</td></tr><tr><td>non-null</td><td>null</td><td>The value is present and may be any type, including null</td></tr><tr><td>null</td><td>non-null</td><td>The value is present and is the shredded type</td>< [...]
+Writers must not produce data where both <code>value</code> and <code>typed_value</code> are non-null, unless the Variant value is an object.</p><p>If a Variant is missing in a context where a value is required, readers must return a Variant null (<code>00</code>): basic type 0 (primitive) and physical type 0 (null).
+For example, if a Variant is required (like <code>measurement</code> above) and both <code>value</code> and <code>typed_value</code> are null, the returned <code>value</code> must be <code>00</code> (Variant null).</p><h3 id=shredded-value-types>Shredded Value Types</h3><p>Shredded values must use the following Parquet types:</p><table><thead><tr><th>Variant Type</th><th>Parquet Physical Type</th><th>Parquet Logical Type</th></tr></thead><tbody><tr><td>boolean</td><td>BOOLEAN</td><td></t [...]
+If the value is an array, <code>value</code> must be null.</p><p>The list <code>element</code> must be a required group.
+The <code>element</code> group can contain <code>value</code> and <code>typed_value</code> fields.
+The element’s <code>value</code> field stores the element as Variant-encoded <code>binary</code> when the <code>typed_value</code> is not present or cannot represent it.
+The <code>typed_value</code> field may be omitted when not shredding elements as a specific type.
+The <code>value</code> field may be omitted when shredding elements as a specific type.
+However, at least one of the two fields must be present.</p><p>For example, a <code>tags</code> Variant may be shredded as a list of strings using the following definition:</p><pre tabindex=0><code>optional group tags (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value (LIST) { # must be optional to allow a null list
+ repeated group list {
+ required group element { # shredded element
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+}
+</code></pre><p>All elements of an array must be present (not missing) because the <code>array</code> Variant encoding does not allow missing elements.
+That is, either <code>typed_value</code> or <code>value</code> (but not both) must be non-null.
+Null elements must be encoded in <code>value</code> as Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>The series of <code>tags</code> arrays <code>["comedy", "drama"], ["horror", null], ["comedy", "drama", "romance"], null</code> would be stored as:</p><table><thead><tr><th>Array</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value...value</code></th><th><code>typed_value...typed_value</code></th></tr></thead><tbody><tr><td><code [...]
+If the value is not an object, <code>typed_value</code> must be null.
+Readers can assume that a value is not an object if <code>typed_value</code> is null and that <code>typed_value</code> field values are correct; that is, readers do not need to read the <code>value</code> column if <code>typed_value</code> fields satisfy the required fields.</p><p>Each shredded field in the <code>typed_value</code> group is represented as a required group that contains optional <code>value</code> and <code>typed_value</code> fields.
+The <code>value</code> field stores the value as Variant-encoded <code>binary</code> when the <code>typed_value</code> cannot represent the field.
+This layout enables readers to skip data based on the field statistics for <code>value</code> and <code>typed_value</code>.
+The <code>typed_value</code> field may be omitted when not shredding fields as a specific type.</p><p>The <code>value</code> column of a partially shredded object must never contain fields represented by the Parquet columns in <code>typed_value</code> (shredded fields).
+Readers may always assume that data is written correctly and that shredded fields in <code>typed_value</code> are not present in <code>value</code>.
+As a result, reads when a field is defined in both <code>value</code> and a <code>typed_value</code> shredded field may be inconsistent.</p><p>For example, a Variant <code>event</code> field may shred <code>event_type</code> (<code>string</code>) and <code>event_ts</code> (<code>timestamp</code>) columns using the following definition:</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value; # a variant, expected to be an object
+ optional group typed_value { # shredded fields for the variant object
+ required group event_type { # shredded field for event_type
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts { # shredded field for event_ts
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ }
+}
+</code></pre><p>The group for each named field must use repetition level <code>required</code>.</p><p>A field’s <code>value</code> and <code>typed_value</code> are set to null (missing) to indicate that the field does not exist in the variant.
+To encode a field that is present with a null value, the <code>value</code> must contain a Variant null: basic type 0 (primitive) and physical type 0 (null).</p><p>When both <code>value</code> and <code>typed_value</code> for a field are non-null, engines should fail.
+If engines choose to read in such cases, then the <code>typed_value</code> column must be used.
+Readers may always assume that data is written correctly and that only <code>value</code> or <code>typed_value</code> is defined.
+As a result, reads when both <code>value</code> and <code>typed_value</code> are defined may be inconsistent with optimized reads that require only one of the columns.</p><p>The table below shows how the series of objects in the first column would be stored:</p><table><thead><tr><th>Event object</th><th><code>value</code></th><th><code>typed_value</code></th><th><code>typed_value.event_type.value</code></th><th><code>typed_value.event_type.typed_value</code></th><th><code>typed_value.eve [...]
+Readers must return an object when <code>typed_value</code> is non-null containing the shredded fields.</p><h2 id=nesting>Nesting</h2><p>The <code>typed_value</code> associated with any Variant <code>value</code> field can be any shredded type, as shown in the sections above.</p><p>For example, the <code>event</code> object above may also shred sub-fields as object (<code>location</code>) or array (<code>tags</code>).</p><pre tabindex=0><code>optional group event (VARIANT) {
+ required binary metadata;
+ optional binary value;
+ optional group typed_value {
+ required group event_type {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ required group event_ts {
+ optional binary value;
+ optional int64 typed_value (TIMESTAMP(true, MICROS));
+ }
+ required group location {
+ optional binary value;
+ optional group typed_value {
+ required group latitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ required group longitude {
+ optional binary value;
+ optional double typed_value;
+ }
+ }
+ }
+ required group tags {
+ optional binary value;
+ optional group typed_value (LIST) {
+ repeated group list {
+ required group element {
+ optional binary value;
+ optional binary typed_value (STRING);
+ }
+ }
+ }
+ }
+ }
+}
+</code></pre><h1 id=data-skipping>Data Skipping</h1><p>Statistics for <code>typed_value</code> columns can be used for file, row group, or page skipping when <code>value</code> is always null (missing).</p><p>When the corresponding <code>value</code> column is all nulls, all values must be the shredded <code>typed_value</code> field’s type.
+Because the type is known, comparisons with values of that type are valid.
+<code>IS NULL</code>/<code>IS NOT NULL</code> and <code>IS NAN</code>/<code>IS NOT NAN</code> filter results are also valid.</p><p>Comparisons with values of other types are not necessarily valid and data should not be skipped.</p><p>Casting behavior for Variant is delegated to processing engines.
+For example, the interpretation of a string as a timestamp may depend on the engine’s SQL session time zone.</p><h2 id=reconstructing-a-shredded-variant>Reconstructing a Shredded Variant</h2><p>It is possible to recover an unshredded Variant value using a recursive algorithm, where the initial call is to <code>construct_variant</code> with the top-level Variant group fields.</p><div class=highlight><pre tabindex=0 style=background-color:#f8f8f8;-moz-tab-size:4;-o-tab-size:4;tab-siz [...]
+</span></span><span style=display:flex><span> <span style=color:#4e9a06>"""Constructs a Variant from value and typed_value"""</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>typed_value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>dict</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#000>object_fields</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>{</span>
+</span></span><span style=display:flex><span> <span style=color:#000>name</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> < [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000;font-weight:700>(</span><span style=color:#000>name</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>field</span><span style=color:#000;font-weight:700>)</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#000>typed_value</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>}</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a partially shredded object</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>),</span> <span style=color:#4e9a06>"partially shredded value must be an object"</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>typed_value</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>keys</span><span style=color:#000;font-weight:700>()</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>isdisjoint</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>value</span><span style=color:#ce5c00;font [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># union the shredded fields and non-shredded fields</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>union</span><span style=color:#000;f [...]
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantObject</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>object_fields</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded array</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded array must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#000>elements</span> <span style=color:#ce5c00;font-weight:700>=</span> <span style=color:#000;font-weight:700>[</span>
+</span></span><span style=display:flex><span> <span style=color:#000>construct_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span><span style=color:#000>value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elem</span><span style=color:#ce5c00;font-weight:700>.</span> [...]
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>for</span> <span style=color:#000>elem</span> <span style=color:#204a87;font-weight:700>in</span> <span style=color:#204a87>list</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#000;font-weight:700>]</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantArray</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>elements</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># this is a shredded primitive</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>assert</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>,</span> <span style=color:#4e9a06>"shredded primitive must not conflict with variant value"</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#000>value</span> <span style=color:#204a87;font-weight:700>is</span> <span style=color:#204a87;font-weight:700>not</span> <span style=color:#204a87;font-weight:700>None</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>metadata</span><span style=color:#000;font-weight:700>,</span> <span style=color:#000>value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>else</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#8f5902;font-style:italic># value is missing</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#204a87;font-weight:700>None</span>
+</span></span><span style=display:flex><span>
+</span></span><span style=display:flex><span><span style=color:#204a87;font-weight:700>def</span> <span style=color:#000>primitive_to_variant</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>:</span> <span style=color:#000>Any</span><span style=color:#000;font-weight:700>):</span> <span style=color:#000>Variant</span><span style=color:#000;font-weight:700>:</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>if</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>int</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantInteger</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>elif</span> <span style=color:#204a87>isinstance</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>,</span> <span style=color:#204a87>str</span><span style=color:#000;font-weight:700>):</span>
+</span></span><span style=display:flex><span> <span style=color:#204a87;font-weight:700>return</span> <span style=color:#000>VariantString</span><span style=color:#000;font-weight:700>(</span><span style=color:#000>typed_value</span><span style=color:#000;font-weight:700>)</span>
+</span></span><span style=display:flex><span> <span style=color:#ce5c00;font-weight:700>...</span>
+</span></span></code></pre></div><h2 id=backward-and-forward-compatibility>Backward and forward compatibility</h2><p>Shredding is an optional feature of Variant, and readers must continue to be able to read a group containing only <code>value</code> and <code>metadata</code> fields.</p><p>Engines that do not write shredded values must be able to read shredded values according to this spec or must fail.</p><p>Different files may contain conflicting shredding schemas.
+That is, files may contain different <code>typed_value</code> columns for the same Variant with incompatible types.
+It may not be possible to infer or specify a single shredded schema that would allow all Parquet files for a table to be read without reconstructing the value as a Variant.</p><div class=td-page-meta__lastmod>Last modified December 9, 2025: <a data-proofer-ignore href=https://github.com/apache/parquet-site/commit/b588d89f68fc261905f80474a4175036a005fec7>Use docs from parquet-format as source of truth (#142) (b588d89)</a></div></div></main></div></div><footer class="td-footer row d-print- [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/index.html b/output/docs/index.html
index 8af4914..a9b0c85 100644
--- a/output/docs/index.html
+++ b/output/docs/index.html
@@ -2,11 +2,11 @@
The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><meta property="og:url" content="/docs/"><meta property="og:site_name" content="Parquet"><meta property="og:title" content="Documentation"><meta property="og:description" content="Welcome to the documentation for Apache Parquet.
The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><meta property="og:locale" content="en"><meta property="og:type" content="website"><meta itemprop=name content="Documentation"><meta itemprop=description content="Welcome to the documentation for Apache Parquet.
The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><meta itemprop=dateModified content="2025-10-28T06:46:33-04:00"><meta itemprop=wordCount content="36"><meta name=twitter:card content="summary"><meta name=twitter:title content="Documentation"><meta name=twitter:description content="Welcome to the documentation for Apache Parquet.
-The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integrity="sha256-IC5zqOLXq9gNDSBgFnZ0ynW7EW9mEpQBTQfQjSOaxVw=" crossorigin=anonymous><link href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c [...]
+The specification for the Apache Parquet file format is hosted in the parquet-format repository. The current implementation status of various features can be found in the implementation status page."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integrity="sha256-Rrz6ZyqyY+C8eprrSr/0jv8BJx3hm0a4Z4Wc498RRt4=" crossorigin=anonymous><link href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top+of+the+p [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Documentation" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current=page>Documentation</li></ol></nav><div class=td-content><h1>Documentation</h1><header class=article-meta></header><p>Welcome to the documentation for Apache Parquet.</p><p>The specific [...]
+<a id=print href=/_print/docs/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class="td-breadcrumbs td-breadcrumbs__single"><ol class=breadcrumb><li class="breadcrumb-item active" aria-current=page>Documentation</li></ol></nav><div class=td-content><h1>Documentation</h1><header class=article-meta></header><p>Welcome to the document [...]
The current implementation status of various features can be found in the <a href=/docs/file-format/implementationstatus/>implementation status</a> page.</p><div class=section-index><hr class=panel-line><div class=entry><h5><a href=/docs/overview/>Overview</a></h5><p>All about Parquet.</p></div><div class=entry><h5><a href=/docs/concepts/>Concepts</a></h5><p>Glossary of relevant terminology.</p></div><div class=entry><h5><a href=/docs/file-format/>File Format</a></h5><p>Documentation abo [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/blog-posts/index.html b/output/docs/learning-resources/blog-posts/index.html
index e161c4a..4f1ffe2 100644
--- a/output/docs/learning-resources/blog-posts/index.html
+++ b/output/docs/learning-resources/blog-posts/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/learning-resources/blog-posts/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-i [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/learning-resources/blog-posts/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/blog-posts/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-i [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/Blog%20Posts/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources/Blog%20Posts?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+mat [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Blog%20Posts" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/blog-posts/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class="breadcrumb-item active" aria-current=page>Blog Posts</li></ol></nav><div cl [...]
+<a id=print href=/_print/docs/learning-resources/blog-posts/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class="breadcrumb-item active" aria-cu [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/index.html b/output/docs/learning-resources/index.html
index 0b03ad8..6dd90be 100644
--- a/output/docs/learning-resources/index.html
+++ b/output/docs/learning-resources/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/learning-resources/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/app [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/learning-resources/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/app [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+sectio [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Resources" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Resources</li></ol></nav><div class=td-content><h1>Resources</h1><div class=lead>Various resources to learn about the Parq [...]
+<a id=print href=/_print/docs/learning-resources/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class="breadcrumb-item active" aria-current=page>Resources</li></ol></nav><div class=td-content><h1>Resources</h1><div class=le [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html b/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html
index cba294a..499484a 100644
--- a/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html
+++ b/output/docs/learning-resources/presentations/conf-2014-parquet-summit-twitter/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/Presentations/conf-2014-parquet-summit-twitter.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources/Presentations?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+ma [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=#CONF%202014" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/presentations/>Presentat [...]
+<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html b/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html
index bfb8cf3..2853e69 100644
--- a/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html
+++ b/output/docs/learning-resources/presentations/hadoop-summit-2014/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/Presentations/hadoop-summit-2014.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources/Presentations?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+ma [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Hadoop%20Summit%202014" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/presentations/>Presentat [...]
+<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/presentations/index.html b/output/docs/learning-resources/presentations/index.html
index b6ed125..91444fc 100644
--- a/output/docs/learning-resources/presentations/index.html
+++ b/output/docs/learning-resources/presentations/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/learning-resources/presentations/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-t [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/learning-resources/presentations/><link rel=alternate type=application/rss+xml href=/docs/learning-resources/presentations/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-t [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/Presentations/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources/Presentations?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+ma [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Presentations" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class="breadcrumb-item active" aria-current=page>Presentations</li></ol></nav>< [...]
+<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class="breadcrumb-item active" aria [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/presentations/spark-summit-2020/index.html b/output/docs/learning-resources/presentations/spark-summit-2020/index.html
index 23b0bcb..b59f2c9 100644
--- a/output/docs/learning-resources/presentations/spark-summit-2020/index.html
+++ b/output/docs/learning-resources/presentations/spark-summit-2020/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/Presentations/spark-summit-2020.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources/Presentations?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+ma [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Spark%20Summit%202020" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/presentations/>Presentat [...]
+<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/learning-resources/presentations/strata-2013/index.html b/output/docs/learning-resources/presentations/strata-2013/index.html
index 78c3782..f1ab3a9 100644
--- a/output/docs/learning-resources/presentations/strata-2013/index.html
+++ b/output/docs/learning-resources/presentations/strata-2013/index.html
@@ -1,7 +1,7 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon-16x16.png sizes=16x16><link rel=icon type=image/png href=/favicon [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Learning%20Resources/Presentations/strata-2013.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Learning%20Resources/Presentations?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+ma [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Strata%202013" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/presentations/>Presentat [...]
+<a id=print href=/_print/docs/learning-resources/presentations/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/learning-resources/>Resources</a></li><li class=breadcrumb-item><a href=/docs [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/overview/index.html b/output/docs/overview/index.html
index cdca2ee..1c940e3 100644
--- a/output/docs/overview/index.html
+++ b/output/docs/overview/index.html
@@ -1,8 +1,9 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/overview/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x18 [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=text/html href=/_print/docs/overview/><link rel=alternate type=application/rss+xml href=/docs/overview/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x18 [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Overview/_index.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Overview?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Overview" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/overview/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc><nav id=TableOfContents><ul><li><ul><li><a href=#parquet-format-specification>parquet-format (Specification)</a></li><li><a href=#parquet-java>parquet-java</a></li><li><a href=#other-clients--libraries--tools>Other Clients / Libraries / Tools</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=bread [...]
+<a id=print href=/_print/docs/overview/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore><div class=td-toc-title><span class=td-toc-title__text>On this page</span>
+<a class=td-toc-title__link title="Top of page" href=#></a></div><nav id=TableOfContents><ul><li><ul><li><a href=#parquet-format-specification>parquet-format (Specification)</a></li><li><a href=#parquet-java>parquet-java</a></li><li><a href=#other-clients--libraries--tools>Other Clients / Libraries / Tools</a></li></ul></li></ul></nav></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=bread [...]
It provides high performance compression and encoding schemes to handle complex data in bulk and is supported in many programming languages and analytics tools.</p><h3 id=parquet-format-specification>parquet-format (Specification)</h3><p>The <a href=https://github.com/apache/parquet-format>parquet-format</a> repository hosts the official specification of the Parquet file format, defining how data is structured and stored. This specification, along with the <a href=https://github.com/apac [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/docs/overview/motivation/index.html b/output/docs/overview/motivation/index.html
index 7a24598..c9014f0 100644
--- a/output/docs/overview/motivation/index.html
+++ b/output/docs/overview/motivation/index.html
@@ -6,10 +6,10 @@ Parquet is built to support very efficient compression and encoding schemes. Mul
Parquet is built from the ground up with complex nested data structures in mind, and uses the record shredding and assembly algorithm described in the Dremel paper. We believe this approach is superior to simple flattening of nested name spaces.
Parquet is built to support very efficient compression and encoding schemes. Multiple projects have demonstrated the performance impact of applying the right compression and encoding scheme to the data. Parquet allows compression schemes to be specified on a per-column level, and is future-proofed to allow adding more encodings as they are invented and implemented."><meta itemprop=dateModified content="2022-03-24T09:39:00-04:00"><meta itemprop=wordCount content="166"><meta name=twitter:c [...]
Parquet is built from the ground up with complex nested data structures in mind, and uses the record shredding and assembly algorithm described in the Dremel paper. We believe this approach is superior to simple flattening of nested name spaces.
-Parquet is built to support very efficient compression and encoding schemes. Multiple projects have demonstrated the performance impact of applying the right compression and encoding scheme to the data. Parquet allows compression schemes to be specified on a per-column level, and is future-proofed to allow adding more encodings as they are invented and implemented."><link rel=preload href=/scss/main.min.202e73a8e2d7abd80d0d2060167674ca75bb116f661294014d07d08d239ac55c.css as=style integri [...]
+Parquet is built to support very efficient compression and encoding schemes. Multiple projects have demonstrated the performance impact of applying the right compression and encoding scheme to the data. Parquet allows compression schemes to be specified on a per-column level, and is future-proofed to allow adding more encodings as they are invented and implemented."><link rel=preload href=/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css as=style integri [...]
<a href=https://github.com/apache/parquet-site/edit/production/content/en/docs/Overview/motivation.md class="td-page-meta--edit td-page-meta__edit" target=_blank rel=noopener><i class="fa-solid fa-pen-to-square fa-fw"></i> Edit this page</a>
<a href="https://github.com/apache/parquet-site/new/production/content/en/docs/Overview?filename=change-me.md&value=---%0Atitle%3A+%22Long+Page+Title%22%0AlinkTitle%3A+%22Short+Nav+Title%22%0Aweight%3A+100%0Adescription%3A+%3E-%0A+++++Page+description+for+heading+and+indexes.%0A---%0A%0A%23%23+Heading%0A%0AEdit+this+template+to+create+your+new+page.%0A%0A%2A+Give+it+a+good+name%2C+ending+in+%60.md%60+-+e.g.+%60getting-started.md%60%0A%2A+Edit+the+%22front+matter%22+section+at+the+top [...]
<a href="https://github.com/apache/parquet-site/issues/new?title=Motivation" class="td-page-meta--issue td-page-meta__issue" target=_blank rel=noopener><i class="fa-solid fa-list-check fa-fw"></i> Create documentation issue</a>
-<a id=print href=/_print/docs/overview/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/overview/>Overview</a></li><li class="breadcrumb-item active" aria-current=page>Motivation</li></ol></nav><div class=td-content><h1>Motivation</h [...]
+<a id=print href=/_print/docs/overview/><i class="fa-solid fa-print fa-fw"></i> Print entire section</a></div><div class=td-toc data-proofer-ignore></div></aside><main class="col-12 col-md-9 col-xl-8 ps-md-5" role=main><nav aria-label=breadcrumb class=td-breadcrumbs><ol class=breadcrumb><li class=breadcrumb-item><a href=/docs/>Documentation</a></li><li class=breadcrumb-item><a href=/docs/overview/>Overview</a></li><li class="breadcrumb-item active" aria-current=page>Motivation</li></ol>< [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/index.html b/output/index.html
index fc0a24c..66b5cf3 100644
--- a/output/index.html
+++ b/output/index.html
@@ -1,5 +1,5 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta name=generator content="Hugo 0.152.0"><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=application/rss+xml href=/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=ico [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta name=generator content="Hugo 0.152.0"><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=application/rss+xml href=/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=ico [...]
</a><a class="btn btn-lg btn-secondary me-3 mb-4" href=/blog/>Releases <i class="fa fa-arrow-circle-down ms-2"></i></a><p class="lead mt-5">Apache Parquet is an open source, column-oriented data file format designed for efficient data storage and retrieval.
It provides high performance compression and encoding schemes to handle complex data in bulk and is supported in many programming languages and analytics tools.</p><p><a class="btn btn-link text-info" href=#td-block-1 aria-label="Read more"><i class="fa-solid fa-circle-chevron-down" style=font-size:400%></i></a></p></div></div></div></div></section><div><a id=td-block-1 class=td-offset-anchor></a></div><section class="row td-box td-box--white td-box--height-auto"><div class=col><div clas [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
diff --git a/output/index.xml b/output/index.xml
index 041f356..7e740fa 100644
--- a/output/index.xml
+++ b/output/index.xml
@@ -1,20 +1,32 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Parquet</title><link>/</link><description>Recent content on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/index.xml" rel="self" type="application/rss+xml"/><item><title>Compression</title><link>/docs/file-format/data-pages/compression/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/da [...]
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom"><channel><title>Parquet</title><link>/</link><description>Recent content on Parquet</description><generator>Hugo</generator><language>en</language><atom:link href="/index.xml" rel="self" type="application/rss+xml"/><item><title/><link>/docs/file-format/binaryprotocolextensions/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/binaryprotocolexte [...]
+<p>The extension mechanism of the <code>binary</code> Thrift field-id <code>32767</code> has some desirable properties:</p>
+<ul>
+<li>Existing readers will ignore these extensions without any modifications</li>
+<li>Existing readers will ignore the extension bytes with little processing overhead</li>
+<li>The content of the extension is freeform and can be encoded in any format. This format is not restricted to Thrift.</li>
+<li>Extensions can be appended to existing Thrift serialized structs <a href="/docs/file-format/binaryprotocolextensions/#appending-extensions-to-thrift">without requiring Thrift libraries</a> for manipulation (or changes to the thrift IDL).</li>
+</ul>
+<p>Because only one field-id is reserved the extension bytes themselves require disambiguation; otherwise readers will not be able to decode extensions safely. This is left to implementers which MUST put enough unique state in their extension bytes for disambiguation. This can be relatively easily achieved by adding a <a href="https://en.wikipedia.org/wiki/Universally%5C_unique%5C_identifier">UUID</a> at the start or end of the extension bytes. The extension does not sp [...]
+<p>This document contains the specification of all supported compression codecs.</p>
+<h2 id="overview">Overview</h2>
<p>Parquet allows the data block inside dictionary pages and data pages to
be compressed for better space efficiency. The Parquet format supports
-several compression covering different areas in the compression ratio /
-processing cost spectrum.</p>
+several compression codecs covering different areas in the compression
+ratio / processing cost spectrum.</p>
<p>The detailed specifications of compression codecs are maintained externally
by their respective authors or maintainers, which we reference hereafter.</p>
<p>For all compression codecs except the deprecated <code>LZ4</code> codec, the raw data
of a (data or dictionary) page is fed <em>as-is</em> to the underlying compression
library, without any additional framing or padding. The information required
for precise allocation of compressed and decompressed buffers is written
-in the <code>PageHeader</code> struct.</p></description></item><item><title>Encodings</title><link>/docs/file-format/data-pages/encodings/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encodings/</guid><description><p><a name="PLAIN"></a></p>
+in the <code>PageHeader</code> struct.</p></description></item><item><title/><link>/docs/file-format/data-pages/encodings/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encodings/</guid><description><h1 id="parquet-encoding-definitions">Parquet encoding definitions</h1>
+<p>This file contains the specification of all supported encodings.</p>
+<p><a name="PLAIN"></a></p>
<h3 id="plain-plain--0">Plain: (PLAIN = 0)</h3>
<p>Supported Types: all</p>
<p>This is the plain encoding that must be supported for types. It is
intended to be the simplest encoding. Values are encoded back to back.</p>
-<p>The plain encoding is used whenever a more efficient encoding cannot be used. It
+<p>The plain encoding is used whenever a more efficient encoding can not be used. It
stores the data in the following format:</p>
<ul>
<li>BOOLEAN: <a href="/docs/file-format/data-pages/encodings/#BITPACKED">Bit Packed</a>, LSB first</li>
@@ -27,7 +39,8 @@ stores the data in the following format:</p>
<li>FIXED_LEN_BYTE_ARRAY: the bytes contained in the array</li>
</ul>
<p>For native types, this outputs the data as little endian. Floating
-point types are encoded in IEEE.</p></description></item><item><title>Parquet Modular Encryption</title><link>/docs/file-format/data-pages/encryption/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encryption/</guid><description><p>Parquet files containing sensitive information can be protected by the modular encryption
+point types are encoded in IEEE.</p></description></item><item><title/><link>/docs/file-format/data-pages/encryption/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/data-pages/encryption/</guid><description><h1 id="parquet-modular-encryption">Parquet Modular Encryption</h1>
+<p>Parquet files containing sensitive information can be protected by the modular encryption
mechanism that encrypts and authenticates the file data and metadata - while allowing
for a regular Parquet functionality (columnar projection, predicate pushdown, encoding
and compression).</p>
@@ -83,13 +96,25 @@ per HDFS file.</p></description></item><item><title>Events</title><link>/d
<li>File Version: The file metadata contains a version.</li>
<li>Encodings: Encodings are specified by enum and more can be added in the future.</li>
<li>Page types: Additional page types can be added and safely skipped.</li>
-</ul></description></item><item><title>Logical Types</title><link>/docs/file-format/types/logicaltypes/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/logicaltypes/</guid><description><p>Logical types are used to extend the types that parquet can be used to store,
+</ul></description></item><item><title/><link>/docs/file-format/types/geospatial/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/geospatial/</guid><description><h1 id="geospatial-definitions">Geospatial Definitions</h1>
+<p>This document contains the specification of geospatial types and statistics.</p>
+<h1 id="background">Background</h1>
+<p>The Geometry and Geography class hierarchy and its Well-Known Text (WKT) and
+Well-Known Binary (WKB) serializations (ISO variant supporting XY, XYZ, XYM,
+XYZM) are defined by <a href="https://portal.ogc.org/files/?artifact_id=25355">OpenGIS Implementation Specification for Geographic
+information - Simple feature access - Part 1: Common architecture</a>,
+from <a href="https://www.ogc.org/standard/sfa/">OGC(Open Geospatial Consortium)</a>.</p>
+<p>The version of the OGC standard first used here is 1.2.1, but future versions
+may also be used if the WKB representation remains wire-compatible.</p></description></item><item><title/><link>/docs/file-format/types/logicaltypes/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/logicaltypes/</guid><description><h1 id="parquet-logical-type-definitions">Parquet Logical Type Definitions</h1>
+<p>Logical types are used to extend the types that parquet can be used to store,
by specifying how the primitive types should be interpreted. This keeps the set
of primitive types to a minimum and reuses parquet&rsquo;s efficient encodings. For
-example, strings are stored as byte arrays (binary) with a UTF8 annotation.
-These annotations define how to further decode and interpret the data.
-Annotations are stored as <code>LogicalType</code> fields in the file metadata and are
-documented in <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md">LogicalTypes.md</a></p></description></item><item><title>Metadata</title><link>/docs/file-format/metadata/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/metadata/</guid><description><p>There are two types of metadata: file metadata, and page header metadata.</p>
+example, strings are stored with the primitive type <code>BYTE_ARRAY</code> with a <code>STRING</code>
+annotation.</p>
+<p>This file contains the specification for all logical types.</p>
+<h3 id="metadata">Metadata</h3>
+<p>The parquet format&rsquo;s <code>LogicalType</code> stores the type annotation. The annotation
+may require additional metadata fields, as well as rules for those fields.</p></description></item><item><title>Metadata</title><link>/docs/file-format/metadata/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/metadata/</guid><description><p>There are two types of metadata: file metadata, and page header metadata.</p>
<p>All thrift structures are serialized using the TCompactProtocol. The full
definition of these structures is given in the Parquet
<a href="https://github.com/apache/parquet-format/blob/master/src/main/thrift/parquet.thrift">Thrift definition</a>.</p>
@@ -100,13 +125,28 @@ when navigating the Parquet file.</p>
<p><img src="/images/FileMetaData.svg" alt="Parquet Metadata format"></p>
<h2 id="page-header">Page header</h2>
<p>Page header metadata (<code>PageHeader</code> and children in the diagram) is stored
-in-line with the page data, and is used in the reading and decoding of data.</p></description></item><item><title>Nested Encoding</title><link>/docs/file-format/nestedencoding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/nestedencoding/</guid><description><p>To encode nested columns, Parquet uses the Dremel encoding with definition and
+in-line with the page data, and is used in the reading and decoding of data.</p></description></item><item><title/><link>/docs/file-format/types/variantshredding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/variantshredding/</guid><description><h1 id="variant-shredding">Variant Shredding</h1>
+<p>The Variant type is designed to store and process semi-structured data efficiently, even with heterogeneous values.
+Query engines encode each Variant value in a self-describing format, and store it as a group containing <code>value</code> and <code>metadata</code> binary fields in Parquet.
+Since data is often partially homogeneous, it can be beneficial to extract certain fields into separate Parquet columns to further improve performance.
+This process is called <strong>shredding</strong>.</p>
+<p>Shredding enables the use of Parquet&rsquo;s columnar representation for more compact data encoding, column statistics for data skipping, and partial projections.</p></description></item><item><title/><link>/docs/file-format/types/variantencoding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/types/variantencoding/</guid><description><h1 id="variant-binary-encoding">Variant Binary Encoding</h1>
+<p>A Variant represents a type that contains one of:</p>
+<ul>
+<li>Primitive: A type and corresponding value (e.g. INT, STRING)</li>
+<li>Array: An ordered list of Variant values</li>
+<li>Object: An unordered collection of string/Variant pairs (i.e. key/value pairs). An object may not contain duplicate keys.</li>
+</ul>
+<p>A Variant is encoded with 2 binary values, the <a href="/docs/file-format/types/variantencoding/#value-encoding">value</a> and the <a href="/docs/file-format/types/variantencoding/#metadata-encoding">metadata</a>.</p>
+<p>There are a fixed number of allowed primitive types, provided in the table below.
+These represent a commonly supported subset of the <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md">logical types</a> allowed by the Parquet format.</p></description></item><item><title>Nested Encoding</title><link>/docs/file-format/nestedencoding/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/nestedencoding/</guid><description><p>To encode nested columns, Parquet uses the Dremel encoding with definition and
repetition levels. Definition levels specify how many optional fields in the
path for the column are defined. Repetition levels specify at what repeated field
in the path has the value repeated. The max definition and repetition levels can
be computed from the schema (i.e. how much nesting there is). This defines the
maximum number of bits required to store the levels (levels are defined for all
-values in the column).</p></description></item><item><title>Bloom Filter</title><link>/docs/file-format/bloomfilter/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/bloomfilter/</guid><description><h3 id="problem-statement">Problem statement</h3>
+values in the column).</p></description></item><item><title/><link>/docs/file-format/bloomfilter/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/bloomfilter/</guid><description><h1 id="parquet-bloom-filter">Parquet Bloom Filter</h1>
+<h3 id="problem-statement">Problem statement</h3>
<p>In their current format, column statistics and dictionaries can be used for predicate
pushdown. Statistics include minimum and maximum value, which can be used to filter out
values not in the range. Dictionaries are more specific, and readers can filter out values
@@ -131,9 +171,12 @@ the data within a page is corrupt, that page is lost. The file will be more
resilient to corruption with smaller row groups.</p></description></item><item><title>Nulls</title><link>/docs/file-format/nulls/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/nulls/</guid><description><p>Nullity is encoded in the definition levels (which is run-length encoded). NULL values
are not encoded in the data. For example, in a non-nested schema, a column with 1000 NULLs
would be encoded with run-length encoding (0, 1000 times) for the definition levels and
-nothing else.</p></description></item><item><title>Page Index</title><link>/docs/file-format/pageindex/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/pageindex/</guid><description><p>This document describes the format for column index pages in the Parquet
-footer. These pages contain statistics for DataPages and can be used to skip
-pages when scanning data in ordered and unordered columns.</p>
+nothing else.</p></description></item><item><title/><link>/docs/file-format/pageindex/</link><pubDate>Mon, 01 Jan 0001 00:00:00 +0000</pubDate><guid>/docs/file-format/pageindex/</guid><description><h1 id="parquet-page-index-layout-to-support-page-skipping">Parquet page index: Layout to Support Page Skipping</h1>
+<p>In Parquet, a <em>page index</em> is optional metadata for a
+ColumnChunk, containing statistics for DataPages that can be used
+to skip those pages when scanning in ordered and unordered columns.
+The page index is stored using the OffsetIndex and ColumnIndex structures,
+defined in <a href="src/main/thrift/parquet.thrift"><code>parquet.thrift</code></a></p>
<h2 id="problem-statement">Problem Statement</h2>
<p>In previous versions of the format, Statistics are stored for ColumnChunks in
ColumnMetaData and for individual pages inside DataPageHeader structs. When
diff --git a/output/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js b/output/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js
new file mode 100644
index 0000000..6cd4a09
--- /dev/null
+++ b/output/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js
@@ -0,0 +1,5 @@
+/*!
+ * Bootstrap v5.3.8 (https://getbootstrap.com/)
+ * Copyright 2011-2025 The Bootstrap Authors (https://github.com/twbs/bootstrap/graphs/contributors)
+ * Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)
+ */(function(e,t){typeof exports=="object"&&typeof module!="undefined"?module.exports=t():typeof define=="function"&&define.amd?define(t):(e=typeof globalThis!="undefined"?globalThis:e||self,e.bootstrap=t())})(this,function(){"use strict";const C=new Map,gt={set(e,t,n){C.has(e)||C.set(e,new Map);const s=C.get(e);if(!s.has(t)&&s.size!==0){console.error(`Bootstrap doesn't allow more than one instance per element. Bound instance: ${Array.from(s.keys())[0]}.`);return}s.set(t,n)},get(e,t){re [...]
\ No newline at end of file
diff --git a/output/parquet-format/doc/images/FileLayoutBloomFilter1.png b/output/parquet-format/doc/images/FileLayoutBloomFilter1.png
new file mode 100644
index 0000000..3b21738
Binary files /dev/null and b/output/parquet-format/doc/images/FileLayoutBloomFilter1.png differ
diff --git a/output/parquet-format/doc/images/FileLayoutBloomFilter2.png b/output/parquet-format/doc/images/FileLayoutBloomFilter2.png
new file mode 100644
index 0000000..6bbf770
Binary files /dev/null and b/output/parquet-format/doc/images/FileLayoutBloomFilter2.png differ
diff --git a/output/parquet-format/doc/images/FileLayoutEncryptionEF.png b/output/parquet-format/doc/images/FileLayoutEncryptionEF.png
new file mode 100644
index 0000000..f8fc3ef
Binary files /dev/null and b/output/parquet-format/doc/images/FileLayoutEncryptionEF.png differ
diff --git a/output/parquet-format/doc/images/FileLayoutEncryptionPF.png b/output/parquet-format/doc/images/FileLayoutEncryptionPF.png
new file mode 100644
index 0000000..b7046a1
Binary files /dev/null and b/output/parquet-format/doc/images/FileLayoutEncryptionPF.png differ
diff --git a/output/parquet-format/doc/images/PageIndexLayout.png b/output/parquet-format/doc/images/PageIndexLayout.png
new file mode 100644
index 0000000..83c5b02
Binary files /dev/null and b/output/parquet-format/doc/images/PageIndexLayout.png differ
diff --git a/output/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css b/output/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css
new file mode 100644
index 0000000..25e1654
--- /dev/null
+++ b/output/scss/main.min.46bcfa672ab263e0bc7a9aeb4abff48eff01271de19b46b867859ce3df1146de.css
@@ -0,0 +1,9 @@
+:root,[data-bs-theme=light]{--td-pre-bg:var(--bs-tertiary-bg)}/*!* Bootstrap v5.3.8 (https://getbootstrap.com/)
+* Copyright 2011-2025 The Bootstrap Authors
+* Licensed under MIT (https://github.com/twbs/bootstrap/blob/main/LICENSE)*/:root,[data-bs-theme=light]{--bs-blue:#0d6efd;--bs-indigo:#6610f2;--bs-purple:#6f42c1;--bs-pink:#d63384;--bs-red:#dc3545;--bs-orange:#fd7e14;--bs-yellow:#ffc107;--bs-green:#198754;--bs-teal:#20c997;--bs-cyan:#0dcaf0;--bs-black:#000;--bs-white:#fff;--bs-gray:#6c757d;--bs-gray-dark:#343a40;--bs-gray-100:#f8f9fa;--bs-gray-200:#e9ecef;--bs-gray-300:#dee2e6;--bs-gray-400:#ced4da;--bs-gray-500:#adb5bd;--bs-gray-600:#6c [...]
+* License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+* Copyright 2024 Fonticons, Inc.*/.fa,.td-search__icon::before{font-family:var(--fa-style-family,"Font Awesome 6 Free");font-weight:var(--fa-style,900)}.fas,.td-offline-search-results__close-button::after,.far,.fab,.fa-solid,.fa-regular,.fa-brands,.fa,.td-search__icon::before{-moz-osx-font-smoothing:grayscale;-webkit-font-smoothing:antialiased;display:var(--fa-display,inline-block);font-style:normal;font-variant:normal;line-height:1;text-rendering:auto}.fas::before,.far::before,.fab::bef [...]
+* License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+* Copyright 2024 Fonticons, Inc.*/:root,:host{--fa-style-family-classic:'Font Awesome 6 Free';--fa-font-solid:normal 900 1em/1 'Font Awesome 6 Free'}@font-face{font-family:'font awesome 6 free';font-style:normal;font-weight:900;font-display:block;src:url(../webfonts/fa-solid-900.woff2)format("woff2"),url(../webfonts/fa-solid-900.ttf)format("truetype")}.fas,.td-offline-search-results__close-button::after,.fa-solid{font-weight:900}/*!* Font Awesome Free 6.7.2 by @fontawesome - https://font [...]
+* License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License)
+* Copyright 2024 Fonticons, Inc.*/:root,:host{--fa-style-family-brands:'Font Awesome 6 Brands';--fa-font-brands:normal 400 1em/1 'Font Awesome 6 Brands'}@font-face{font-family:'font awesome 6 brands';font-style:normal;font-weight:400;font-display:block;src:url(../webfonts/fa-brands-400.woff2)format("woff2"),url(../webfonts/fa-brands-400.ttf)format("truetype")}.fab,.fa-brands{font-weight:400}.fa-monero{--fa:"\f3d0"}.fa-hooli{--fa:"\f427"}.fa-yelp{--fa:"\f1e9"}.fa-cc-visa{--fa:"\f1f0"}.fa- [...]
\ No newline at end of file
diff --git a/output/sitemap.xml b/output/sitemap.xml
index 872349d..929ed40 100644
--- a/output/sitemap.xml
+++ b/output/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"><url><loc>/docs/file-format/data-pages/compression/</loc><lastmod>2024-03-11T22:11:10+01:00</lastmod></url><url><loc>/docs/file-format/data-pages/encodings/</loc><lastmod>2025-11-12T06:31:27-05:00</lastmod></url><url><loc>/docs/file-format/data-pages/encryption/</loc><lastmod>2025-11-12T06:31:27-05:00</lastmod></url><url><loc>/docs/ [...]
\ No newline at end of file
+<?xml version="1.0" encoding="utf-8" standalone="yes"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml"><url><loc>/docs/file-format/binaryprotocolextensions/</loc><lastmod>2025-12-09T11:38:22-08:00</lastmod></url><url><loc>/docs/file-format/data-pages/compression/</loc><lastmod>2025-12-09T11:38:22-08:00</lastmod></url><url><loc>/docs/file-format/data-pages/encodings/</loc><lastmod>2025-12-09T11:38:22-08:00</lastmod></url><url><loc>/do [...]
\ No newline at end of file
diff --git a/output/tags/index.html b/output/tags/index.html
index 3e684ba..85b7ae1 100644
--- a/output/tags/index.html
+++ b/output/tags/index.html
@@ -1,3 +1,3 @@
-<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=application/rss+xml href=/tags/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon [...]
+<!doctype html><html itemscope itemtype=http://schema.org/WebPage lang=en class=no-js><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1,shrink-to-fit=no"><link rel=alternate type=application/rss+xml href=/tags/index.xml><meta name=robots content="index, follow"><link rel="shortcut icon" href=/favicons/favicon.ico><link rel=apple-touch-icon href=/favicons/apple-touch-icon-180x180.png sizes=180x180><link rel=icon type=image/png href=/favicons/favicon [...]
2025
-<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.5ea9bb4146d5e591d8c4d97ce08831bb4f2ea2dbf02ed68c5ecd03b7fa7f2bb4.js integrity="sha256-Xqm7QUbV5ZHYxNl84Igxu08uotvwLtaMXs0Dt/p/K7Q=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file
+<span class=td-footer__authors>Apache Parquet</span></span><span class=td-footer__all_rights_reserved>All Rights Reserved</span><span class=ms-2><a href=https://policies.google.com/privacy target=_blank rel=noopener>Privacy Policy</a></span></div></div></div></footer></div><script src=/js/main.min.127f48c815760e8c1346474ec609632241944fd383a1b8a5f849b29478b9283a.js integrity="sha256-En9IyBV2DowTRkdOxgljIkGUT9ODobil+EmylHi5KDo=" crossorigin=anonymous></script><script defer src=/js/click-to [...]
\ No newline at end of file