You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2019/07/06 20:08:24 UTC
[incubator-iceberg] 04/06: Deployed 601de88e with MkDocs version:
1.0.4
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
commit ad0237a389b1f3f6e80648754f2844a450b32b9e
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Sun Jun 23 12:11:18 2019 -0800
Deployed 601de88e with MkDocs version: 1.0.4
---
404.html | 6 +
community/index.html | 6 +
css/extra.css | 4 +
404.html => getting-started/index.html | 59 +-
img/iceberg-metadata.png | Bin 0 -> 15937 bytes
index.html | 8 +-
partitioning/index.html | 10 +-
search/search_index.json | 2 +-
sitemap.xml | 11 +-
sitemap.xml.gz | Bin 200 -> 202 bytes
spec/index.html | 1701 ++++++++++++++++++++++++++++++++
11 files changed, 1774 insertions(+), 33 deletions(-)
diff --git a/404.html b/404.html
index 70d3eb6..494b3bb 100644
--- a/404.html
+++ b/404.html
@@ -99,6 +99,12 @@
</li>
+
+<li >
+ <a href="/spec/">Spec</a>
+</li>
+
+
</ul>
</li>
diff --git a/community/index.html b/community/index.html
index 7136f73..20895ce 100644
--- a/community/index.html
+++ b/community/index.html
@@ -99,6 +99,12 @@
</li>
+
+<li >
+ <a href="../spec/">Spec</a>
+</li>
+
+
</ul>
</li>
diff --git a/css/extra.css b/css/extra.css
index 898f792..fa7952b 100644
--- a/css/extra.css
+++ b/css/extra.css
@@ -27,3 +27,7 @@
margin-left: -1.6em;
float: left;
}
+
+.floating {
+ float: right;
+}
diff --git a/404.html b/getting-started/index.html
similarity index 83%
copy from 404.html
copy to getting-started/index.html
index 70d3eb6..630573f 100644
--- a/404.html
+++ b/getting-started/index.html
@@ -8,21 +8,21 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="/img/favicon.ico">
+ <link rel="../img/favicon.ico">
- <title>Apache Iceberg (incubating)</title>
+ <title>Getting started - Apache Iceberg (incubating)</title>
<link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
<link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="/css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="/css/base.min.css" rel="stylesheet">
- <link href="/css/cinder.min.css" rel="stylesheet">
- <link href="/css/highlight.min.css" rel="stylesheet">
- <link href="/css/extra.css" rel="stylesheet">
+ <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="../css/base.min.css" rel="stylesheet">
+ <link href="../css/cinder.min.css" rel="stylesheet">
+ <link href="../css/highlight.min.css" rel="stylesheet">
+ <link href="../css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -62,7 +62,7 @@
<!-- Main title -->
- <a class="navbar-brand" href="/.">Apache Iceberg (incubating)</a>
+ <a class="navbar-brand" href="..">Apache Iceberg (incubating)</a>
</div>
<!-- Expanded navigation -->
@@ -72,13 +72,13 @@
<li >
- <a href="/.">About</a>
+ <a href="..">About</a>
</li>
<li >
- <a href="/community/">Community</a>
+ <a href="../community/">Community</a>
</li>
@@ -89,7 +89,7 @@
<li >
- <a href="/partitioning/">Partitioning</a>
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -99,6 +99,12 @@
</li>
+
+<li >
+ <a href="../spec/">Spec</a>
+</li>
+
+
</ul>
</li>
@@ -118,16 +124,17 @@
<div class="container">
-
- <div class="row-fluid">
- <div id="main-content" class="span12">
- <h1 id="404-page-not-found" style="text-align: center">404</h1>
- <p style="text-align: center"><strong>Page not found</strong></p>
- <p style="text-align: center"><a href="/">Home</a></p>
- </div>
- </div>
-
-
+
+ <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+ <ul class="nav bs-sidenav">
+ <li class="first-level active"><a href="#getting-started">Getting Started</a></li>
+ </ul>
+</div></div>
+ <div class="col-md-9" role="main">
+
+<h2 id="getting-started">Getting Started</h2></div>
+
+
</div>
<footer class="col-md-12 text-center">
@@ -140,13 +147,13 @@
</footer>
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="/js/bootstrap-3.0.3.min.js"></script>
- <script src="/js/highlight.pack.js"></script>
+ <script src="../js/bootstrap-3.0.3.min.js"></script>
+ <script src="../js/highlight.pack.js"></script>
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = "/"</script>
+ <script>var base_url = ".."</script>
- <script src="/js/base.js"></script>
- <script src="/search/main.js"></script>
+ <script src="../js/base.js"></script>
+ <script src="../search/main.js"></script>
<div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
<div class="modal-dialog modal-lg">
diff --git a/img/iceberg-metadata.png b/img/iceberg-metadata.png
new file mode 100644
index 0000000..b3146d3
Binary files /dev/null and b/img/iceberg-metadata.png differ
diff --git a/index.html b/index.html
index 361e999..9753b07 100644
--- a/index.html
+++ b/index.html
@@ -99,6 +99,12 @@
</li>
+
+<li >
+ <a href="spec/">Spec</a>
+</li>
+
+
</ul>
</li>
@@ -288,5 +294,5 @@
<!--
MkDocs version : 1.0.4
-Build Date UTC : 2019-06-23 01:40:20
+Build Date UTC : 2019-06-23 20:11:18
-->
diff --git a/partitioning/index.html b/partitioning/index.html
index 95fe338..69b3f95 100644
--- a/partitioning/index.html
+++ b/partitioning/index.html
@@ -99,6 +99,12 @@
</li>
+
+<li >
+ <a href="../spec/">Spec</a>
+</li>
+
+
</ul>
</li>
@@ -116,8 +122,8 @@
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
- <li class="disabled">
- <a rel="next" >
+ <li >
+ <a rel="next" href="../spec/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
diff --git a/search/search_index.json b/search/search_index.json
index 269fb14..17df0b6 100644
--- a/search/search_index.json
+++ b/search/search_index.json
@@ -1 +1 @@
-{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Apache Iceberg is a new table format for large, slow-moving tabular data. It is designed to improve on the de-facto standard table layout built into Hive, Presto, and Spark. Iceberg Overview Iceberg tracks individual data files in a table instead of directories. This allows writers to create data files in-place and only adds files to the table in an explicit commit. Table state is main [...]
\ No newline at end of file
+{"config":{"lang":["en"],"prebuild_index":false,"separator":"[\\s\\-]+"},"docs":[{"location":"","text":"Apache Iceberg is a new table format for large, slow-moving tabular data. It is designed to improve on the de-facto standard table layout built into Hive, Presto, and Spark. Iceberg Overview Iceberg tracks individual data files in a table instead of directories. This allows writers to create data files in-place and only adds files to the table in an explicit commit. Table state is main [...]
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 782c7fd..1ea99c5 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -2,17 +2,17 @@
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
<url>
<loc>None</loc>
- <lastmod>2019-06-22</lastmod>
+ <lastmod>2019-06-23</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2019-06-22</lastmod>
+ <lastmod>2019-06-23</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
<loc>None</loc>
- <lastmod>2019-06-22</lastmod>
+ <lastmod>2019-06-23</lastmod>
<changefreq>daily</changefreq>
</url>
<url>
@@ -20,4 +20,9 @@
<changefreq>daily</changefreq>
</url>
+ <url>
+ <loc>None</loc>
+ <lastmod>2019-06-23</lastmod>
+ <changefreq>daily</changefreq>
+ </url>
</urlset>
\ No newline at end of file
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 6f590b1..758e476 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ
diff --git a/spec/index.html b/spec/index.html
new file mode 100644
index 0000000..ee9beb2
--- /dev/null
+++ b/spec/index.html
@@ -0,0 +1,1701 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <meta name="description" content="A table format for large, slow-moving tabular data">
+
+
+ <link rel="../img/favicon.ico">
+
+
+ <title>Spec - Apache Iceberg (incubating)</title>
+
+
+ <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
+ <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="../css/base.min.css" rel="stylesheet">
+ <link href="../css/cinder.min.css" rel="stylesheet">
+ <link href="../css/highlight.min.css" rel="stylesheet">
+ <link href="../css/extra.css" rel="stylesheet">
+
+ <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+ <!--[if lt IE 9]>
+ <script src="https://cdn.jsdelivr.net/npm/html5shiv@3.7.3/dist/html5shiv.min.js"></script>
+ <script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
+ <![endif]-->
+
+ <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
+ <script>
+ WebFont.load({
+ google: {
+ families: ['Open Sans', 'PT Sans']
+ }
+ });
+ </script>
+
+
+
+
+</head>
+
+<body>
+
+ <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+ <div class="container">
+
+ <!-- Collapsed navigation -->
+ <div class="navbar-header">
+ <!-- Expander button -->
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
+
+ <!-- Main title -->
+
+ <a class="navbar-brand" href="..">Apache Iceberg (incubating)</a>
+ </div>
+
+ <!-- Expanded navigation -->
+ <div class="navbar-collapse collapse">
+ <!-- Main navigation -->
+ <ul class="nav navbar-nav">
+
+
+ <li >
+ <a href="..">About</a>
+ </li>
+
+
+
+ <li >
+ <a href="../community/">Community</a>
+ </li>
+
+
+
+ <li class="dropdown active">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Docs <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="../partitioning/">Partitioning</a>
+</li>
+
+
+
+<li >
+ <a href="/javadoc/">Javadoc</a>
+</li>
+
+
+
+<li class="active">
+ <a href="./">Spec</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+ </ul>
+
+ <ul class="nav navbar-nav navbar-right">
+ <li>
+ <a href="#" data-toggle="modal" data-target="#mkdocs_search_modal">
+ <i class="fas fa-search"></i> Search
+ </a>
+ </li>
+ <li >
+ <a rel="prev" href="../partitioning/">
+ <i class="fas fa-arrow-left"></i> Previous
+ </a>
+ </li>
+ <li class="disabled">
+ <a rel="next" >
+ Next <i class="fas fa-arrow-right"></i>
+ </a>
+ </li>
+ </ul>
+ </div>
+ </div>
+</div>
+
+ <div class="container">
+
+
+ <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+ <ul class="nav bs-sidenav">
+ <li class="first-level active"><a href="#iceberg-table-spec">Iceberg Table Spec</a></li>
+ <li class="second-level"><a href="#goals">Goals</a></li>
+
+ <li class="second-level"><a href="#overview">Overview</a></li>
+
+ <li class="third-level"><a href="#mvcc-and-optimistic-concurrency">MVCC and Optimistic Concurrency</a></li>
+ <li class="third-level"><a href="#file-system-operations">File System Operations</a></li>
+ <li class="second-level"><a href="#specification">Specification</a></li>
+
+ <li class="third-level"><a href="#terms">Terms</a></li>
+ <li class="third-level"><a href="#schemas-and-data-types">Schemas and Data Types</a></li>
+ <li class="third-level"><a href="#primitive-types">Primitive Types</a></li>
+ <li class="third-level"><a href="#schema-evolution">Schema Evolution</a></li>
+ <li class="second-level"><a href="#partitioning">Partitioning</a></li>
+
+ <li class="third-level"><a href="#partition-transforms">Partition Transforms</a></li>
+ <li class="third-level"><a href="#bucket-transform-details">Bucket Transform Details</a></li>
+ <li class="third-level"><a href="#truncate-transform-details">Truncate Transform Details</a></li>
+ <li class="second-level"><a href="#manifests">Manifests</a></li>
+
+ <li class="third-level"><a href="#manifest-entry-fields">Manifest Entry Fields</a></li>
+ <li class="second-level"><a href="#snapshots">Snapshots</a></li>
+
+ <li class="third-level"><a href="#scan-planning">Scan Planning</a></li>
+ <li class="third-level"><a href="#manifest-lists">Manifest Lists</a></li>
+ <li class="second-level"><a href="#table-metadata">Table Metadata</a></li>
+
+ <li class="third-level"><a href="#commit-conflict-resolution-and-retry">Commit Conflict Resolution and Retry</a></li>
+ <li class="third-level"><a href="#table-metadata-fields">Table Metadata Fields</a></li>
+ <li class="third-level"><a href="#file-system-tables">File System Tables</a></li>
+ <li class="third-level"><a href="#metastore-tables">Metastore Tables</a></li>
+ <li class="first-level "><a href="#appendix-a-format-specific-requirements">Appendix A: Format-specific Requirements</a></li>
+ <li class="second-level"><a href="#avro">Avro</a></li>
+
+ <li class="third-level"><a href="#data-type-mappings">Data Type Mappings</a></li>
+ <li class="third-level"><a href="#field-ids">Field IDs</a></li>
+ <li class="second-level"><a href="#parquet">Parquet</a></li>
+
+ <li class="third-level"><a href="#data-type-mappings_1">Data Type Mappings</a></li>
+ <li class="second-level"><a href="#orc">ORC</a></li>
+
+ <li class="first-level "><a href="#appendix-b-32-bit-hash-requirements-by-type">Appendix B: 32-bit Hash Requirements by Type</a></li>
+ <li class="first-level "><a href="#appendix-c-json-serialization">Appendix C: JSON serialization</a></li>
+ <li class="second-level"><a href="#schemas">Schemas</a></li>
+
+ <li class="second-level"><a href="#partition-specs">Partition Specs</a></li>
+
+ <li class="second-level"><a href="#table-metadata-and-snapshots">Table Metadata and Snapshots</a></li>
+
+ <li class="first-level "><a href="#appendix-d-single-value-serialization">Appendix D: Single-value serialization</a></li>
+ </ul>
+</div></div>
+ <div class="col-md-9" role="main">
+
+<h1 id="iceberg-table-spec">Iceberg Table Spec</h1>
+<p>This is a specification for the Iceberg table format that is designed to manage a large, slow-changing collection of files in a distributed file system or key-value store as a table.</p>
+<h2 id="goals">Goals</h2>
+<ul>
+<li><strong>Snapshot isolation</strong> – Reads will be isolated from concurrent writes and always use a committed snapshot of a table’s data. Writes will support removing and adding files in a single operation and are never partially visible. Readers will not acquire locks.</li>
+<li><strong>Speed</strong> – Operations will use O(1) remote calls to plan the files for a scan and not O(n) where n grows with the size of the table, like the number of partitions or files.</li>
+<li><strong>Scale</strong> – Job planning will be handled primarily by clients and not bottleneck on a central metadata store. Metadata will include information needed for cost-based optimization.</li>
+<li><strong>Evolution</strong> – Tables will support full schema and partition spec evolution. Schema evolution supports safe column add, drop, reorder and rename, including in nested structures.</li>
+<li><strong>Dependable types</strong> – Tables will provide well-defined and dependable support for a core set of types.</li>
+<li><strong>Storage separation</strong> – Partitioning will be table configuration. Reads will be planned using predicates on data values, not partition values. Tables will support evolving partition schemes.</li>
+<li><strong>Formats</strong> – Underlying data file formats will support identical schema evolution rules and types. Both read- and write-optimized formats will be available.</li>
+</ul>
+<h2 id="overview">Overview</h2>
+<p><img alt="Iceberg snapshot structure" class="floating" src="../img/iceberg-metadata.png" /></p>
+<p>This table format tracks individual data files in a table instead of directories. This allows writers to create data files in-place and only adds files to the table in an explicit commit.</p>
+<p>Table state is maintained in metadata files. All changes to table state create a new metadata file and replace the old metadata with an atomic swap. The table metadata file tracks the table schema, partitioning config, custom properties, and snapshots of the table contents. A snapshot represents the state of a table at some time and is used to access the complete set of data files in the table.</p>
+<p>Data files in snapshots are tracked by one or more manifest files that contain a row for each data file in the table, the file’s partition data, and its metrics. The data in a snapshot is the union of all files in its manifests. Manifest files are reused across snapshots to avoid rewriting metadata that is slow-changing. Manifests can track data files with any subset of a table and are not associated with partitions.</p>
+<p>The manifests that make up a snapshot are stored in a manifest list file. Each manifest list stores metadata about manifests, including partition stats and data file counts. These stats are used to avoid reading manifests that are not required for an operation.</p>
+<h3 id="mvcc-and-optimistic-concurrency">MVCC and Optimistic Concurrency</h3>
+<p>An atomic swap of one table metadata file for another provides serializable isolation. Readers use the snapshot that was current when they load the table metadata and are not affected by changes until they refresh and pick up a new metadata location.</p>
+<p>Writers create table metadata files optimistically, assuming that the current version will not be changed before the writer’s commit. Once a writer has created an update, it commits by swapping the table’s metadata file pointer from the base version to the new version.</p>
+<p>If the snapshot on which an update is based is no longer current, the writer must retry the update based on the new current version. Some operations support retry by re-applying metadata changes and committing, under well-defined conditions. For example, a change that rewrites files can be applied to a new table snapshot if all of the rewritten files are still in the table.</p>
+<h3 id="file-system-operations">File System Operations</h3>
+<p>Iceberg only requires that file systems support the following operations:</p>
+<ul>
+<li><strong>In-place write</strong>: files are not moved or altered once they are written</li>
+<li><strong>Seekable reads</strong>: data file formats require seek support</li>
+<li><strong>Deletes</strong>: tables delete files that are no longer used</li>
+</ul>
+<p>These requirements are compatible with object stores, like S3.</p>
+<p>Tables do not require random-access writes. Once written, data and metadata files are immutable until they are deleted.</p>
+<p>Tables do not require rename, except fo rtables that use atomic rename to implement the commit operation for new metadata files.</p>
+<h2 id="specification">Specification</h2>
+<h3 id="terms">Terms</h3>
+<ul>
+<li><strong>Schema</strong> – names and types of fields in a table</li>
+<li><strong>Partition spec</strong> – a definition of how partition values are derived from data fields</li>
+<li><strong>Snapshot</strong> – the state of a table at some point in time, including the set of all data files</li>
+<li><strong>Manifest</strong> – a file that lists data files; a subset of a snapshot</li>
+<li><strong>Manifest list</strong> – a file that lists manifest files; one per snapshot</li>
+</ul>
+<h3 id="schemas-and-data-types">Schemas and Data Types</h3>
+<p>A table’s <strong>schema</strong> is a list of named columns. All data types are either primitives or nested types, which are maps, lists, or structs. A table schema is also a struct type.</p>
+<p>For the representations of these types in Avro, ORC, and Parquet file formats, see Appendix A.</p>
+<h4 id="nested-types">Nested Types</h4>
+<p>A <strong><code>struct</code></strong> is a tuple of typed values. Each field in the tuple is named and has an integer id that is unique in the table schema. Each field can be either optional or required, meaning that values can (or cannot) be null. Fields may be any type. Fields may have an optional comment or doc string.</p>
+<p>A <strong><code>list</code></strong> is a collection of values with some element type. The element field has an integer id that is unique in the table schema. Elements can be either optional or required. Element types may be any type.</p>
+<p>A <strong><code>map</code></strong> is a collection of key-value pairs with a key type and a value type. Both the key field and value field each have an integer id that is unique in the table schema. Map keys are required and map values can be either optional or required. Both map keys and map values may be any type, including nested types.</p>
+<h3 id="primitive-types">Primitive Types</h3>
+<table>
+<thead>
+<tr>
+<th>Primitive type</th>
+<th>Description</th>
+<th>Requirements</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td>True or false</td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td>32-bit signed integers</td>
+<td>Can promote to <code>long</code></td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td>64-bit signed integers</td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td><a href="https://en.wikipedia.org/wiki/IEEE_754">32-bit IEEE 754</a> floating point</td>
+<td>Can promote to double</td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td><a href="https://en.wikipedia.org/wiki/IEEE_754">64-bit IEEE 754</a> floating point</td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>decimal(P,S)</code></strong></td>
+<td>Fixed-point decimal; precision P, scale S</td>
+<td>Scale is fixed [1], Precision must be 38 or less</td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td>Calendar date without timezone or time</td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td>Time of day without date, timezone</td>
+<td>Microsecond precision [2]</td>
+</tr>
+<tr>
+<td><strong><code>timestamp</code></strong></td>
+<td>Timestamp without timezone</td>
+<td>Microsecond precision [2]</td>
+</tr>
+<tr>
+<td><strong><code>timestamptz</code></strong></td>
+<td>Timestamp with timezone</td>
+<td>Stored as UTC [2]</td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td>Arbitrary-length character sequences</td>
+<td>Encoded with UTF-8 [3]</td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td>Universally unique identifiers</td>
+<td>Should use 16-byte fixed</td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td>Fixed-length byte array of length L</td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td>Arbitrary-length byte array</td>
+<td></td>
+</tr>
+</tbody>
+</table>
+<p>Notes:</p>
+<ol>
+<li>Decimal scale is fixed and cannot be changed by schema evolution. Precision can only be widened.</li>
+<li>All time and timestamp values are stored with microsecond precision.
+Timestamps <em>with time zone</em> represent a point in time: values are stored as UTC and do not retain a source time zone (<code>2017-11-16 17:10:34 PST</code> is stored/retrieved as <code>2017-11-17 01:10:34 UTC</code> and these values are considered identical).
+Timestamps <em>without time zone</em> represent a date and time of day regardless of zone: the time value is independent of zone adjustments (<code>2017-11-16 17:10:34</code> is always retrieved as <code>2017-11-16 17:10:34</code>). Timestamp values are stored as a long that encodes microseconds from the unix epoch.</li>
+<li>Character strings must be stored as UTF-8 encoded byte arrays.</li>
+</ol>
+<p>For details on how to serialize a schema to JSON, see Appendix C.</p>
+<h3 id="schema-evolution">Schema Evolution</h3>
+<p>Schema evolution is limited to type promotion and adding, deleting, and renaming fields in structs (both nested structs and the top-level schema’s struct).</p>
+<p>Valid type promotions are:</p>
+<ul>
+<li><code>int</code> to <code>long</code></li>
+<li><code>float</code> to <code>double</code></li>
+<li><code>decimal(P, S)</code> to <code>decimal(P', S)</code> if <code>P' > P</code> – widen the precision of decimal types.</li>
+</ul>
+<p>Any struct, including a top-level schema, can evolve through deleting fields, adding new fields, renaming existing fields, or promoting a primitive using the valid type promotions. Adding a new field assigns a new ID for that field and for any nested fields. Renaming an existing field must change the name, but not the field ID. Deleting a field removes it from the current schema. Field deletion cannot be rolled back unless the field was nullable or if the current snapshot has not chan [...]
+<p>Grouping a subset of a struct’s fields into a nested struct is <strong>not</strong> allowed, nor is moving fields from a nested struct into its immediate parent struct (<code>struct<a, b, c> ↔ struct<a, struct<b, c>></code>). Evolving primitive types to structs is <strong>not</strong> allowed, nor is evolving a single-field struct to a primitive (<code>map<string, int> ↔ map<string, struct<int>></code>).</p>
+<h2 id="partitioning">Partitioning</h2>
+<p>Data files are stored in manifests with a tuple of partition values that are used in scans to filter out files that cannot contain records that match the scan’s filter predicate. Partition values for a data file must be the same for all records stored in the data file. (Manifests store data files from any partition, as long as the partition spec is the same for the data files.)</p>
+<p>Tables are configured with a <strong>partition spec</strong> that defines how to produce a tuple of partition values from a record. A partition spec has a list of fields that consist of:</p>
+<ul>
+<li>A <strong>source column id</strong> from the table’s schema</li>
+<li>A <strong>transform</strong> that is applied to the source column to produce a partition value</li>
+<li>A <strong>partition name</strong></li>
+</ul>
+<p>The source column, selected by id, must be a primitive type and cannot be contained in a map or list, but may be nested in a struct. For details on how to serialize a partition spec to JSON, see Appendix C.</p>
+<p>Partition specs capture the transform from table data to partition values. This is used to transform predicates to partition predicates, in addition to transforming data values. Deriving partition predicates from column predicates on the table data is used to separate the logical queries from physical storage: the partitioning can change and the correct partition filters are always derived from column predicates. This simplifies queries because users don’t have to supply both logical [...]
+<h3 id="partition-transforms">Partition Transforms</h3>
+<table>
+<thead>
+<tr>
+<th>Transform</th>
+<th>Description</th>
+<th>Source types</th>
+<th>Result type</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>identity</code></strong></td>
+<td>Source value, unmodified</td>
+<td>Any</td>
+<td>Source type</td>
+</tr>
+<tr>
+<td><strong><code>bucket[N]</code></strong></td>
+<td>Hash of value, mod <code>N</code> (see below)</td>
+<td><code>int</code>, <code>long</code>, <code>decimal</code>, <code>date</code>, <code>time</code>, <code>timestamp</code>, <code>timestamptz</code>, <code>string</code>, <code>uuid</code>, <code>fixed</code>, <code>binary</code></td>
+<td><code>int</code></td>
+</tr>
+<tr>
+<td><strong><code>truncate[W]</code></strong></td>
+<td>Value truncated to width <code>W</code> (see below)</td>
+<td><code>int</code>, <code>long</code>, <code>decimal</code>, <code>string</code></td>
+<td>Source type</td>
+</tr>
+<tr>
+<td><strong><code>year</code></strong></td>
+<td>Extract a date or timestamp year, as years from 1970</td>
+<td><code>date</code>, <code>timestamp(tz)</code></td>
+<td><code>int</code></td>
+</tr>
+<tr>
+<td><strong><code>month</code></strong></td>
+<td>Extract a date or timestamp month, as months from 1970-01-01</td>
+<td><code>date</code>, <code>timestamp(tz)</code></td>
+<td><code>int</code></td>
+</tr>
+<tr>
+<td><strong><code>day</code></strong></td>
+<td>Extract a date or timestamp day, as days from 1970-01-01</td>
+<td><code>date</code>, <code>timestamp(tz)</code></td>
+<td><code>int</code></td>
+</tr>
+<tr>
+<td><strong><code>hour</code></strong></td>
+<td>Extract a timestamp hour, as hours from 1970-01-01 00:00:00</td>
+<td><code>timestamp(tz)</code></td>
+<td><code>int</code></td>
+</tr>
+</tbody>
+</table>
+<p>All transforms must return <code>null</code> for a <code>null</code> input value.</p>
+<h3 id="bucket-transform-details">Bucket Transform Details</h3>
+<p>Bucket partition transforms use a 32-bit hash of the source value. The 32-bit hash implementation is the 32-bit Murmur3 hash, x86 variant, seeded with 0.</p>
+<p>Transforms are parameterized by a number of buckets[^3], <code>N</code>. The hash mod <code>N</code> must produce a positive value by first discarding the sign bit of the hash value. In pseudo-code, the function is:</p>
+<pre><code> def bucket_N(x) = (murmur3_x86_32_hash(x) & Integer.MAX_VALUE) % N
+</code></pre>
+
+<p>For hash function details by type, see Appendix B.</p>
+<h3 id="truncate-transform-details">Truncate Transform Details</h3>
+<table>
+<thead>
+<tr>
+<th><strong>Type</strong></th>
+<th><strong>Config</strong></th>
+<th><strong>Truncate specification</strong></th>
+<th><strong>Examples</strong></th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td><code>W</code>, width</td>
+<td><code>v - (v % W)</code> remainders must be positive [1]</td>
+<td><code>W=10</code>: <code>1</code> ￫ <code>0</code>, <code>-1</code> ￫ <code>-10</code></td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td><code>W</code>, width</td>
+<td><code>v - (v % W)</code> remainders must be positive [1]</td>
+<td><code>W=10</code>: <code>1</code> ￫ <code>0</code>, <code>-1</code> ￫ <code>-10</code></td>
+</tr>
+<tr>
+<td><strong><code>decimal</code></strong></td>
+<td><code>W</code>, width (no scale)</td>
+<td><code>scaled_W = decimal(W, scale(v))</code> <code>v - (v % scaled_W)</code> [1, 2]</td>
+<td><code>W=50</code>, <code>s=2</code>: <code>10.65</code> ￫ <code>10.50</code></td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td><code>L</code>, length</td>
+<td>Substring of length <code>L</code>: <code>v.substring(0, L)</code></td>
+<td><code>L=3</code>: <code>iceberg</code> ￫ <code>ice</code></td>
+</tr>
+</tbody>
+</table>
+<p>Notes:</p>
+<ol>
+<li>The remainder, <code>v % W</code>, must be positive. For languages where <code>%</code> can produce negative values, the correct truncate function is: <code>v - (((v % W) + W) % W)</code></li>
+<li>The width, <code>W</code>, used to truncate decimal values is applied using the scale of the decimal column to avoid additional (and potentially conflicting) parameters.</li>
+</ol>
+<h2 id="manifests">Manifests</h2>
+<p>A manifest is an immutable Avro file that lists a set of data files, along with each file’s partition data tuple, metrics, and tracking information. One or more manifest files are used to store a snapshot, which tracks all of the files in a table at some point in time.</p>
+<p>A manifest is a valid Iceberg data file. Files must use Iceberg schemas and column projection.</p>
+<p>A manifest stores files for a single partition spec. When a table’s partition spec changes, old files remain in the older manifest and newer files are written to a new manifest. This is required because a manifest file’s schema is based on its partition spec (see below). This restriction also simplifies selecting files from a manifest because the same boolean expression can be used to select or filter all rows.</p>
+<p>The partition spec for a manifest and the current table schema must be stored in the key-value properties of the manifest file. The partition spec is stored as a JSON string under the key <code>partition-spec</code>. The table schema is stored as a JSON string under the key <code>schema</code>.</p>
+<p>The schema of a manifest file is a struct called <code>manifest_entry</code> with the following fields:</p>
+<table>
+<thead>
+<tr>
+<th>Field id, name</th>
+<th>Type</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>0 status</code></strong></td>
+<td><code>int</code> with meaning: <code>0: EXISTING</code> <code>1: ADDED</code> <code>2: DELETED</code></td>
+<td>Used to track additions and deletions</td>
+</tr>
+<tr>
+<td><strong><code>1 snapshot_id</code></strong></td>
+<td><code>long</code></td>
+<td>Snapshot id where the file was added, or deleted if status is 2</td>
+</tr>
+<tr>
+<td><strong><code>2 data_file</code></strong></td>
+<td><code>data_file</code> <code>struct</code> (see below)</td>
+<td>File path, partition tuple, metrics, …</td>
+</tr>
+</tbody>
+</table>
+<p><code>data_file</code> is a struct with the following fields:</p>
+<table>
+<thead>
+<tr>
+<th>Field id, name</th>
+<th>Type</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>100 file_path</code></strong></td>
+<td><code>string</code></td>
+<td>Full URI for the file with FS scheme</td>
+</tr>
+<tr>
+<td><strong><code>101 file_format</code></strong></td>
+<td><code>string</code></td>
+<td>String file format name, avro, orc or parquet</td>
+</tr>
+<tr>
+<td><strong><code>102 partition</code></strong></td>
+<td><code>struct<...></code></td>
+<td>Partition data tuple, schema based on the partition spec</td>
+</tr>
+<tr>
+<td><strong><code>103 record_count</code></strong></td>
+<td><code>long</code></td>
+<td>Number of records in this file</td>
+</tr>
+<tr>
+<td><strong><code>104 file_size_in_bytes</code></strong></td>
+<td><code>long</code></td>
+<td>Total file size in bytes</td>
+</tr>
+<tr>
+<td><del><strong><code>105 block_size_in_bytes</code></strong></del></td>
+<td><code>long</code></td>
+<td><strong>Deprecated. Always write a default value and do not read.</strong></td>
+</tr>
+<tr>
+<td><strong><code>106 file_ordinal</code></strong></td>
+<td><code>optional int</code></td>
+<td>Ordinal of the file w.r.t files with the same partition tuple and snapshot id</td>
+</tr>
+<tr>
+<td><strong><code>107 sort_columns</code></strong></td>
+<td><code>optional list</code></td>
+<td>Columns the file is sorted by</td>
+</tr>
+<tr>
+<td><strong><code>108 column_sizes</code></strong></td>
+<td><code>optional map</code></td>
+<td>Map from column id to the total size on disk of all regions that store the column. Does not include bytes necessary to read other columns, like footers. Leave null for row-oriented formats (Avro).</td>
+</tr>
+<tr>
+<td><strong><code>109 value_counts</code></strong></td>
+<td><code>optional map</code></td>
+<td>Map from column id to number of values in the column (including null values)</td>
+</tr>
+<tr>
+<td><strong><code>110 null_value_counts</code></strong></td>
+<td><code>optional map</code></td>
+<td>Map from column id to number of null values in the column</td>
+</tr>
+<tr>
+<td><del><strong><code>111 distinct_counts</code></strong></del></td>
+<td><code>optional map</code></td>
+<td><strong>Deprecated. Do not use.</strong></td>
+</tr>
+<tr>
+<td><strong><code>125 lower_bounds</code></strong></td>
+<td><code>optional map<126: int, 127: binary></code></td>
+<td>Map from column id to lower bound in the column serialized as binary [1]. Each value must be less than or equal to all values in the column for the file.</td>
+</tr>
+<tr>
+<td><strong><code>128 upper_bounds</code></strong></td>
+<td><code>optional map<129: int, 130: binary></code></td>
+<td>Map from column id to upper bound in the column serialized as binary [1]. Each value must be greater than or equal to all values in the column for the file.</td>
+</tr>
+<tr>
+<td><strong><code>131 key_metadata</code></strong></td>
+<td><code>optional binary</code></td>
+<td>Implementation-specific key metadata for encryption</td>
+</tr>
+<tr>
+<td><strong><code>132 split_offsets</code></strong></td>
+<td><code>optional list</code></td>
+<td>Split offsets for the data file. For example, all row group offsets in a Parquet file. Must be sorted ascending.</td>
+</tr>
+</tbody>
+</table>
+<p>Notes:</p>
+<ol>
+<li>Single-value serialization for lower and upper bounds is detailed in Appendix D.</li>
+</ol>
+<p>The <code>partition</code> struct stores the tuple of partition values for each file. Its type is derived from the partition fields of the partition spec for the manifest file.</p>
+<p>Each manifest file must store its partition spec and the current table schema in the Avro file’s key-value metadata. The partition spec is used to transform predicates on the table’s data rows into predicates on the manifest’s partition values during job planning.</p>
+<h3 id="manifest-entry-fields">Manifest Entry Fields</h3>
+<p>The manifest entry fields are used to keep track of the snapshot in which files were added or logically deleted. The <code>data_file</code> struct is nested inside of the manifest entry so that it can be easily passed to job planning without the manifest entry fields.</p>
+<p>When a data file is added to the dataset, it’s manifest entry should store the snapshot ID in which the file was added and set status to 1 (added).</p>
+<p>When a data file is replaced or deleted from the dataset, it’s manifest entry fields store the snapshot ID in which the file was deleted and status 2 (deleted). The file may be deleted from the file system when the snapshot in which it was deleted is garbage collected, assuming that older snapshots have also been garbage collected[^4].</p>
+<h2 id="snapshots">Snapshots</h2>
+<p>A snapshot consists of the following fields:</p>
+<ul>
+<li><strong><code>snapshot-id</code></strong>: a unique long ID.</li>
+<li><strong><code>parent-snapshot-id</code></strong>: (optional) the snapshot ID of the snapshot’s parent. This field is not present for snapshots that have no parent snapshot, such as snapshots created before this field was added or the first snapshot of a table.</li>
+<li><strong><code>timestamp-ms</code></strong>: a timestamp when the snapshot was created. This is used when garbage collecting snapshots.</li>
+<li><strong><code>manifests</code></strong>: a list of manifest file locations. The data files in a snapshot are the union of all data files listed in these manifests. (Deprecated in favor of <code>manifest-list</code>)</li>
+<li><strong><code>manifest-list</code></strong>: (optional) the location of a manifest list file for this snapshot, which contains a list of manifest files with additional metadata. If present, the manifests field must be omitted.</li>
+<li><strong><code>summary</code></strong>: (optional) a summary that encodes the <code>operation</code> that produced the snapshot and other relevant information specific to that operation. This allows some operations like snapshot expiration to skip processing some snapshots. Possible values of <code>operation</code> are:<ul>
+<li><code>append</code>: data files were added and no files were removed.</li>
+<li><code>replace</code>: data files were rewritten with the same data; i.e., compaction, changing the data file format, or relocating data files.</li>
+<li><code>overwrite</code>: data files were deleted and added in a logical overwrite operation.</li>
+<li><code>delete</code>: data files were removed and their contents logically deleted.</li>
+</ul>
+</li>
+</ul>
+<p>Snapshots can be split across more than one manifest. This enables:</p>
+<ul>
+<li>Appends can add a new manifest to minimize the amount of data written, instead of adding new records by rewriting and appending to an existing manifest. (This is called a “fast append”.)</li>
+<li>Tables can use multiple partition specs. A table’s partition configuration can evolve if, for example, its data volume changes. Each manifest uses a single partition spec, and queries do not need to change because partition filters are derived from data predicates.</li>
+<li>Large tables can be split across multiple manifests so that implementations can parallelize job planning or reduce the cost of rewriting a manifest.</li>
+</ul>
+<p>Valid snapshots are stored as a list in table metadata. For serialization, see Appendix C.</p>
+<h3 id="scan-planning">Scan Planning</h3>
+<p>Scans are planned by reading the manifest files for the current snapshot listed in the table metadata. Deleted entries in a manifest are not included in the scan.</p>
+<p>For each manifest, scan predicates, that filter data rows, are converted to partition predicates, that filter data files, and used to select the data files in the manifest. This conversion uses the partition spec used to write the manifest file.</p>
+<p>Scan predicates are converted to partition predicates using an inclusive projection: if a scan predicate matches a row, then the partition predicate must match that row’s partition. This is an <em>inclusive projection</em>[^5] because rows that do not match the scan predicate may be included in the scan by the partition predicate.</p>
+<p>For example, an <code>events</code> table with a timestamp column named <code>ts</code> that is partitioned by <code>ts_day=day(ts)</code> is queried by users with ranges over the timestamp column: <code>ts > X</code>. The inclusive projection is <code>ts_day >= day(X)</code>, which is used to select files that may have matching rows. Note that, in most cases, timestamps just before <code>X</code> will be included in the scan because the file contains rows that match the predica [...]
+<h3 id="manifest-lists">Manifest Lists</h3>
+<p>Snapshots are embedded in table metadata, but the list of manifests for a snapshot can be stored in a separate manifest list file.</p>
+<p>A manifest list encodes extra fields that can be used to avoid scanning all of the manifests in a snapshot when planning a table scan. </p>
+<p>Manifest list files store <code>manifest_file</code>, a struct with the following fields:</p>
+<table>
+<thead>
+<tr>
+<th>Field id, name</th>
+<th>Type</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>500 manifest_path</code></strong></td>
+<td><code>string</code></td>
+<td>Location of the manifest file</td>
+</tr>
+<tr>
+<td><strong><code>501 manifest_length</code></strong></td>
+<td><code>long</code></td>
+<td>Length of the manifest file</td>
+</tr>
+<tr>
+<td><strong><code>502 partition_spec_id</code></strong></td>
+<td><code>int</code></td>
+<td>ID of a partition spec for the table; must be listed in table metadata <code>partition-specs</code></td>
+</tr>
+<tr>
+<td><strong><code>503 added_snapshot_id</code></strong></td>
+<td><code>long</code></td>
+<td>ID of the snapshot where the manifest file was added</td>
+</tr>
+<tr>
+<td><strong><code>504 added_files_count</code></strong></td>
+<td><code>int</code></td>
+<td>Number of entries in the manifest that have status <code>ADDED</code> (1)</td>
+</tr>
+<tr>
+<td><strong><code>505 existing_files_count</code></strong></td>
+<td><code>int</code></td>
+<td>Number of entries in the manifest that have status <code>EXISTING</code> (0)</td>
+</tr>
+<tr>
+<td><strong><code>506 deleted_files_count</code></strong></td>
+<td><code>int</code></td>
+<td>Number of entries in the manifest that have status <code>DELETED</code> (2)</td>
+</tr>
+<tr>
+<td><strong><code>507 partitions</code></strong></td>
+<td><code>list<508: field_summary></code> (see below)</td>
+<td>A list of field summaries for each partition field in the spec. Each field in the list corresponds to a field in the manifest file’s partition spec.</td>
+</tr>
+</tbody>
+</table>
+<p><code>field_summary</code> is a struct with the following fields</p>
+<table>
+<thead>
+<tr>
+<th>Field id, name</th>
+<th>Type</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>509 contains_null</code></strong></td>
+<td><code>boolean</code></td>
+<td>Whether the manifest contains at least one partition with a null value for the field</td>
+</tr>
+<tr>
+<td><strong><code>510 lower_bound</code></strong></td>
+<td><code>optional bytes</code> [1]</td>
+<td>Lower bound for the non-null values in the partition field, or null if all values are null.</td>
+</tr>
+<tr>
+<td><strong><code>511 upper_bound</code></strong></td>
+<td><code>optional bytes</code> [1]</td>
+<td>Upper bound for the non-null values in the partition field, or null if all values are null.</td>
+</tr>
+</tbody>
+</table>
+<p>Notes:</p>
+<ol>
+<li>Lower and upper bounds are serialized to bytes using the single-object serialization in Appendix D. The type of used to encode the value is the type of the partition field data.</li>
+</ol>
+<h2 id="table-metadata">Table Metadata</h2>
+<p>Table metadata is stored as JSON. Each table metadata change creates a new table metadata file that is committed by an atomic operation. This operation is used to ensure that a new version of table metadata replaces the version on which it was based. This produces a linear history of table versions and ensures that concurrent writes are not lost.</p>
+<p>The atomic operation used to commit metadata depends on how tables are tracked and is not standardized by this spec. See the sections below for examples.</p>
+<h3 id="commit-conflict-resolution-and-retry">Commit Conflict Resolution and Retry</h3>
+<p>When two commits happen at the same time and are based on the same version, only one commit will succeed. In most cases, the failed commit can be applied to the new current version of table metadata and retried. Updates verify the conditions under which they can be applied to a new version and retry if those conditions are met.</p>
+<ul>
+<li>Append operations have no requirements and can always be applied.</li>
+<li>Replace operations must verify that the files that will be deleted are still in the table. Examples of replace operations include format changes (replace an Avro file with a Parquet file) and compactions (several files are replaced with a single file that contains the same rows).</li>
+<li>Delete operations must verify that specific files to delete are still in the table. Delete operations based on expressions can always be applied (e.g., where timestamp < X).</li>
+<li>Table schema updates and partition spec changes must validate that the schema has not changed between the base version and the current version.</li>
+</ul>
+<h3 id="table-metadata-fields">Table Metadata Fields</h3>
+<p>Table metadata consists of the following fields:</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>format-version</code></strong></td>
+<td>An integer version number for the format. Currently, this is always 1.</td>
+</tr>
+<tr>
+<td><strong><code>location</code></strong></td>
+<td>The table’s base location. This is used by writers to determine where to store data files, manifest files, and table metadata files.</td>
+</tr>
+<tr>
+<td><strong><code>last-updated-ms</code></strong></td>
+<td>Timestamp in milliseconds from the unix epoch when the table was last updated. Each table metadata file should update this field just before writing.</td>
+</tr>
+<tr>
+<td><strong><code>last-column-id</code></strong></td>
+<td>An integer; the highest assigned column ID for the table. This is used to ensure columns are always assigned an unused ID when evolving schemas.</td>
+</tr>
+<tr>
+<td><strong><code>schema</code></strong></td>
+<td>The table’s current schema.</td>
+</tr>
+<tr>
+<td><strong><code>partition-spec</code></strong></td>
+<td>The table’s current partition spec, stored as only fields. Note that this is used by writers to partition data, but is not used when reading because reads use the specs stored in manifest files. (<strong>Deprecated</strong>: use <code>partition-specs</code> and <code>default-spec-id</code>instead )</td>
+</tr>
+<tr>
+<td><strong><code>partition-specs</code></strong></td>
+<td>A list of partition specs, stored as full partition spec objects.</td>
+</tr>
+<tr>
+<td><strong><code>default-spec-id</code></strong></td>
+<td>ID of the “current” spec that writers should use by default.</td>
+</tr>
+<tr>
+<td><strong><code>properties</code></strong></td>
+<td>A string to string map of table properties. This is used to control settings that affect reading and writing and is not intended to be used for arbitrary metadata. For example, <code>commit.retry.num-retries</code> is used to control the number of commit retries.</td>
+</tr>
+<tr>
+<td><strong><code>current-snapshot-id</code></strong></td>
+<td><code>long</code> ID of the current table snapshot.</td>
+</tr>
+<tr>
+<td><strong><code>snapshots</code></strong></td>
+<td>A list of valid snapshots. Valid snapshots are snapshots for which all data files exist in the file system. A data file must not be deleted from the file system until the last snapshot in which it was listed is garbage collected.</td>
+</tr>
+<tr>
+<td><strong><code>snapshot-log</code></strong></td>
+<td>A list (optional) of timestamp and snapshot ID pairs that encodes changes to the current snapshot for the table. Each time the current-snapshot-id is changed, a new entry should be added with the last-updated-ms and the new current-snapshot-id. When snapshots are expired from the list of valid snapshots, all entries before a snapshot that has expired should be removed.</td>
+</tr>
+</tbody>
+</table>
+<p>For serialization details, see Appendix C.</p>
+<h3 id="file-system-tables">File System Tables</h3>
+<p>An atomic swap can be implemented using atomic rename in file systems that support it, like HDFS or most local file systems[^6].</p>
+<p>Each version of table metadata is stored in a metadata folder under the table’s base location using a file naming scheme that includes a version number, <code>V</code>: <code>v<V>.metadata.json</code>. To commit a new metadata version, <code>V+1</code>, the writer performs the following steps:</p>
+<ol>
+<li>Read the current table metadata version <code>V</code>.</li>
+<li>Create new table metadata based on version <code>V</code>.</li>
+<li>Write the new table metadata to a unique file: <code><random-uuid>.metadata.json</code>.</li>
+<li>Rename the unique file to the well-known file for version <code>V</code>: <code>v<V+1>.metadata.json</code>.<ol>
+<li>If the rename succeeds, the commit succeeded and <code>V+1</code> is the table’s current version</li>
+<li>If the rename fails, go back to step 1.</li>
+</ol>
+</li>
+</ol>
+<h3 id="metastore-tables">Metastore Tables</h3>
+<p>The atomic swap needed to commit new versions of table metadata can be implemented by storing a pointer in a metastore or database that is updated with a check-and-put operation[^7]. The check-and-put validates that the version of the table that a write is based on is still current and then makes the new metadata from the write the current version.</p>
+<p>Each version of table metadata is stored in a metadata folder under the table’s base location using a naming scheme that includes a version and UUID: <code><V>-<uuid>.metadata.json</code>. To commit a new metadata version, <code>V+1</code>, the writer performs the following steps:</p>
+<ol start="2">
+<li>Create a new table metadata file based on the current metadata.</li>
+<li>Write the new table metadata to a unique file: <code><V+1>-<uuid>.metadata.json</code>.</li>
+<li>Request that the metastore swap the table’s metadata pointer from the location of <code>V</code> to the location of <code>V+1</code>.<ol>
+<li>If the swap succeeds, the commit succeeded. <code>V</code> was still the latest metadata version and the metadata file for <code>V+1</code> is now the current metadata.</li>
+<li>If the swap fails, another writer has already created <code>V+1</code>. The current writer goes back to step 1.</li>
+</ol>
+</li>
+</ol>
+<h1 id="appendix-a-format-specific-requirements">Appendix A: Format-specific Requirements</h1>
+<h2 id="avro">Avro</h2>
+<h3 id="data-type-mappings">Data Type Mappings</h3>
+<p>Values should be stored in Avro using the Avro types and logical type annotations in the table below.</p>
+<p>Optional fields, array elements, and map values must be wrapped in an Avro <code>union</code> with <code>null</code>. This is the only union type allowed in Iceberg data files.</p>
+<p>Optional fields must always set the Avro field default value to null.</p>
+<p>Maps with non-string keys must use an array representation with the <code>map</code> logical type. The array representation or Avro’s map type may be used for maps with string keys.</p>
+<table>
+<thead>
+<tr>
+<th>Type</th>
+<th>Avro type</th>
+<th>Notes</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td><code>boolean</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td><code>int</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td><code>long</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td><code>float</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td><code>double</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>decimal(P,S)</code></strong></td>
+<td><code>{ "type": "fixed",</code><br /> <code>"size": minBytesRequired(P),</code><br /> <code>"logicalType": "decimal",</code><br /> <code>"precision": P,</code><br /> <code>"scale": S }</code></td>
+<td>Stored as fixed using the minimum number of bytes for the given precision.</td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td><code>{ "type": "int",</code><br /> <code>"logicalType": "date" }</code></td>
+<td>Stores days from the 1970-01-01</td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td><code>{ "type": "long",</code><br /> <code>"logicalType": "time-micros" }</code></td>
+<td>Stores microseconds from midnight</td>
+</tr>
+<tr>
+<td><strong><code>timestamp</code></strong></td>
+<td><code>{ "type": "long",</code><br /> <code>"logicalType": "timestamp-micros",</code><br /> <code>"adjust-to-utc": false }</code></td>
+<td>Stores microseconds from 1970-01-01 00:00:00.000000</td>
+</tr>
+<tr>
+<td><strong><code>timestamptz</code></strong></td>
+<td><code>{ "type": "long",</code><br /> <code>"logicalType": "timestamp-micros",</code><br /> <code>"adjust-to-utc": true }</code></td>
+<td>Stores microseconds from 1970-01-01 00:00:00.000000 UTC</td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td><code>string</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td><code>{ "type": "fixed",</code><br /> <code>"size": 16,</code><br /> <code>"logicalType": "uuid" }</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td><code>{ "type": "fixed",</code><br /> <code>"size": L }</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td><code>bytes</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>struct</code></strong></td>
+<td><code>record</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>list</code></strong></td>
+<td><code>array</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>map</code></strong></td>
+<td><code>array</code> of key-value records, or <code>map</code> when keys are strings (optional)</td>
+<td>Array storage must use logical type name <code>map</code> and must store elements that are 2-field records. The first field is a non-null key and the second field is the value.</td>
+</tr>
+</tbody>
+</table>
+<h3 id="field-ids">Field IDs</h3>
+<p>Iceberg struct, list, and map types identify nested types by ID. When writing data to Avro files, these IDs must be stored in the Avro schema to support ID-based column pruning.</p>
+<p>IDs are stored as JSON integers in the following locations:</p>
+<table>
+<thead>
+<tr>
+<th>ID</th>
+<th>Avro schema location</th>
+<th>Property</th>
+<th>Example</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong>Struct field</strong></td>
+<td>Record field object</td>
+<td><code>field-id</code></td>
+<td><code>{ "type": "record", ...</code><br /> <code>"fields": [</code><br /> <code>{ "name": "l",</code><br /> <code>"type": ["null", "long"],</code><br /> <code>"default": null,</code><br /> <code>"field-id": 8 }</code><br /> <code>] }</code></td>
+</tr>
+<tr>
+<td><strong>List element</strong></td>
+<td>Array schema object</td>
+<td><code>element-id</code></td>
+<td><code>{ "type": "array",</code><br /> <code>"items": "int",</code><br /> <code>"element-id": 9 }</code></td>
+</tr>
+<tr>
+<td><strong>String map key</strong></td>
+<td>Map schema object</td>
+<td><code>key-id</code></td>
+<td><code>{ "type": "map",</code><br /> <code>"values": "int",</code><br /> <code>"key-id": 10,</code><br /> <code>"value-id": 11 }</code></td>
+</tr>
+<tr>
+<td><strong>String map value</strong></td>
+<td>Map schema object</td>
+<td><code>value-id</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong>Map key, value</strong></td>
+<td>Key, value fields in the element record.</td>
+<td><code>field-id</code></td>
+<td><code>{ "type": "array",</code><br /> <code>"logicalType": "map",</code><br /> <code>"items": {</code><br /> <code>"type": "record",</code><br /> <code>"name": "k12_v13",</code><br /> <code>"fields": [</code><br /> <code>{ "name": "key",</code><br /> <code>"type": "int",</code><br /> & [...]
+</tr>
+</tbody>
+</table>
+<p>Note that the string map case is for maps where the key type is a string. Using Avro’s map type in this case is optional. Maps with string keys may be stored as arrays.</p>
+<h2 id="parquet">Parquet</h2>
+<h3 id="data-type-mappings_1">Data Type Mappings</h3>
+<p>Values should be stored in Parquet using the types and logical type annotations in the table below. Column IDs are required.</p>
+<p>Lists must use the <a href="https://github.com/apache/parquet-format/blob/master/LogicalTypes.md#lists">3-level representation</a>.</p>
+<table>
+<thead>
+<tr>
+<th>Type</th>
+<th>Parquet physical type</th>
+<th>Logical type</th>
+<th>Notes</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td><code>boolean</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td><code>int</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td><code>long</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td><code>float</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td><code>double</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>decimal(P,S)</code></strong></td>
+<td><code>P <= 9</code>: <code>int32</code>,<br /><code>P <= 18</code>: <code>int64</code>,<br /><code>fixed</code> otherwise</td>
+<td><code>DECIMAL(P,S)</code></td>
+<td>Fixed must use the minimum number of bytes that can store <code>P</code>.</td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td><code>int32</code></td>
+<td><code>DATE</code></td>
+<td>Stores days from the 1970-01-01</td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td><code>int64</code></td>
+<td><code>TIME_MICROS</code> with <code>adjustToUtc=false</code></td>
+<td>Stores microseconds from midnight</td>
+</tr>
+<tr>
+<td><strong><code>timestamp</code></strong></td>
+<td><code>int64</code></td>
+<td><code>TIMESTAMP_MICROS</code> with <code>adjustToUtc=false</code></td>
+<td>Stores microseconds from 1970-01-01 00:00:00.000000</td>
+</tr>
+<tr>
+<td><strong><code>timestamptz</code></strong></td>
+<td><code>int64</code></td>
+<td><code>TIMESTAMP_MICROS</code> with <code>adjustToUtc=true</code></td>
+<td>Stores microseconds from 1970-01-01 00:00:00.000000 UTC</td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td><code>binary</code></td>
+<td><code>UTF8</code></td>
+<td>Encoding must be UTF-8</td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td><code>fixed_len_byte_array[16]</code></td>
+<td><code>UUID</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td><code>fixed_len_byte_array[L]</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td><code>binary</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>struct</code></strong></td>
+<td><code>group</code></td>
+<td></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>list</code></strong></td>
+<td><code>3-level list</code></td>
+<td><code>LIST</code></td>
+<td>See Parquet docs for 3-level representation</td>
+</tr>
+<tr>
+<td><strong><code>map</code></strong></td>
+<td><code>3-level map</code></td>
+<td><code>MAP</code></td>
+<td>See Parquet docs for 3-level representation</td>
+</tr>
+</tbody>
+</table>
+<h2 id="orc">ORC</h2>
+<table>
+<thead>
+<tr>
+<th>Type</th>
+<th>ORC type</th>
+<th>Notes</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td><code>boolean</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td><code>int</code></td>
+<td>ORC tinyint and smallint would map to int also.</td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td><code>long</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td><code>float</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td><code>double</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>decimal(P,S)</code></strong></td>
+<td><code>decimal</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td><code>date</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td><code>int</code></td>
+<td>Stores microseconds from midnight</td>
+</tr>
+<tr>
+<td><strong><code>timestamp</code></strong></td>
+<td><code>timestamp</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>timestamptz</code></strong></td>
+<td><code>struct</code></td>
+<td>We should add this to ORC’s type model. (ORC-294)</td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td><code>string</code></td>
+<td>ORC varchar and char would map to Iceberg string too.</td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td><code>binary</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td><code>binary</code></td>
+<td>The length would not be checked by the ORC reader and should be checked by the adaptor.</td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td><code>binary</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>struct</code></strong></td>
+<td><code>struct</code></td>
+<td>ORC uniontype would map to struct also.</td>
+</tr>
+<tr>
+<td><strong><code>list</code></strong></td>
+<td><code>array</code></td>
+<td></td>
+</tr>
+<tr>
+<td><strong><code>map</code></strong></td>
+<td><code>map</code></td>
+<td></td>
+</tr>
+</tbody>
+</table>
+<p>One of the interesting challenges with this is how to map Iceberg’s schema evolution (id based) on to ORC’s (name based). In theory we could use Iceberg’s column ids as the column and field names, but that would suck from a user’s point of view. </p>
+<p>The column ids would be stored in ORC’s user metadata as “iceberg.column.id” with a comma separated list of the ids.</p>
+<p>Iceberg would build the desired reader schema with their schema evolution rules and pass that down to the ORC reader, which would then use its schema evolution to map that to the writer’s schema. Basically, Iceberg would need to change the names of columns and fields to get the desired mapping.</p>
+<table>
+<thead>
+<tr>
+<th>Iceberg writer</th>
+<th>ORC writer</th>
+<th>Iceberg reader</th>
+<th>ORC reader</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>struct<a (1): int, b (2): string></code></td>
+<td><code>struct<a: int, b: string></code></td>
+<td><code>struct<a (2): string, c (3): date></code></td>
+<td><code>struct<b: string, c: date></code></td>
+</tr>
+<tr>
+<td><code>struct<a (1): struct<b (2): string, c (3): date>></code></td>
+<td><code>struct<a: struct<b:string, c:date>></code></td>
+<td><code>struct<aa (1): struct<cc (3): date, bb (2): string>></code></td>
+<td><code>struct<a: struct<c:date, b:string>></code></td>
+</tr>
+</tbody>
+</table>
+<h1 id="appendix-b-32-bit-hash-requirements-by-type">Appendix B: 32-bit Hash Requirements by Type</h1>
+<p>The 32-bit hash implementation is 32-bit Murmur3 hash, x86 variant, seeded with 0.</p>
+<table>
+<thead>
+<tr>
+<th>Primitive type</th>
+<th>Hash specification</th>
+<th>Test value</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td><code>false: hashInt(0)</code>, <code>true: hashInt(1)</code></td>
+<td><code>true</code> ￫ <code>1392991556</code></td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td><code>hashLong(long(v))</code> [1]</td>
+<td><code>34</code> ￫ <code>2017239379</code></td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td><code>hashBytes(littleEndianBytes(v))</code></td>
+<td><code>34L</code> ￫ <code>2017239379</code></td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td><code>hashDouble(double(v))</code> [2]</td>
+<td><code>1.0F</code> ￫ <code>-142385009</code></td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td><code>hashLong(doubleToRawLongBits(v))</code></td>
+<td><code>1.0D</code> ￫ <code>-142385009</code></td>
+</tr>
+<tr>
+<td><strong><code>decimal(P,S)</code></strong></td>
+<td><code>hashBytes(minBigEndian(unscaled(v)))</code>[3]</td>
+<td><code>14.20</code> ￫ <code>-500754589</code></td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td><code>hashInt(daysFromUnixEpoch(v))</code></td>
+<td><code>2017-11-16</code> ￫ <code>-653330422</code></td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td><code>hashLong(microsecsFromMidnight(v))</code></td>
+<td><code>22:31:08</code> ￫ <code>-662762989</code></td>
+</tr>
+<tr>
+<td><strong><code>timestamp</code></strong></td>
+<td><code>hashLong(microsecsFromUnixEpoch(v))</code></td>
+<td><code>2017-11-16T22:31:08</code> ￫ <code>-2047944441</code></td>
+</tr>
+<tr>
+<td><strong><code>timestamptz</code></strong></td>
+<td><code>hashLong(microsecsFromUnixEpoch(v))</code></td>
+<td><code>2017-11-16T14:31:08-08:00</code>￫ <code>-2047944441</code></td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td><code>hashBytes(utf8Bytes(v))</code></td>
+<td><code>iceberg</code> ￫ <code>1210000089</code></td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td><code>hashBytes(uuidBytes(v))</code> [4]</td>
+<td><code>f79c3e09-677c-4bbd-a479-3f349cb785e7</code> ￫ <code>1488055340</code></td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td><code>hashBytes(v)</code></td>
+<td><code>00 01 02 03</code> ￫ <code>188683207</code></td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td><code>hashBytes(v)</code></td>
+<td><code>00 01 02 03</code> ￫ <code>188683207</code></td>
+</tr>
+</tbody>
+</table>
+<p>Notes:</p>
+<ol>
+<li>Integer and long hash results must be identical for all integer values. This ensures that schema evolution does not change bucket partition values if integer types are promoted.</li>
+<li>Float hash values are the result of hashing the float cast to double to ensure that schema evolution does not change hash values if float types are promoted. Note that floating point types are not valid source values for partitioning.</li>
+<li>Decimal values are hashed using the minimum number of bytes required to hold the unscaled value as a two’s complement big-endian; this representation does not include padding bytes required for storage in a fixed-length array.
+Hash results are not dependent on decimal scale, which is part of the type, not the data value.</li>
+<li>UUIDs are encoded using big endian. The test UUID for the example above is: <code>f79c3e09-677c-4bbd-a479-3f349cb785e7</code>. This UUID encoded as a byte array is:
+<code>F7 9C 3E 09 67 7C 4B BD A4 79 3F 34 9C B7 85 E7</code></li>
+</ol>
+<h1 id="appendix-c-json-serialization">Appendix C: JSON serialization</h1>
+<h2 id="schemas">Schemas</h2>
+<p>Schemas are serialized to JSON as a struct. Types are serialized according to this table:</p>
+<table>
+<thead>
+<tr>
+<th>Type</th>
+<th>JSON representation</th>
+<th>Example</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td><code>JSON string: "boolean"</code></td>
+<td><code>"boolean"</code></td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td><code>JSON string: "int"</code></td>
+<td><code>"int"</code></td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td><code>JSON string: "long"</code></td>
+<td><code>"long"</code></td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td><code>JSON string: "float"</code></td>
+<td><code>"float"</code></td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td><code>JSON string: "double"</code></td>
+<td><code>"double"</code></td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td><code>JSON string: "date"</code></td>
+<td><code>"date"</code></td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td><code>JSON string: "time"</code></td>
+<td><code>"time"</code></td>
+</tr>
+<tr>
+<td><strong><code>timestamp without zone</code></strong></td>
+<td><code>JSON string: "timestamp"</code></td>
+<td><code>"timestamp"</code></td>
+</tr>
+<tr>
+<td><strong><code>timestamp with zone</code></strong></td>
+<td><code>JSON string: "timestamptz"</code></td>
+<td><code>"timestamptz"</code></td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td><code>JSON string: "string"</code></td>
+<td><code>"string"</code></td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td><code>JSON string: "uuid"</code></td>
+<td><code>"uuid"</code></td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td><code>JSON string: "fixed[<L>]"</code></td>
+<td><code>"fixed[16]"</code></td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td><code>JSON string: "binary"</code></td>
+<td><code>"binary"</code></td>
+</tr>
+<tr>
+<td><strong><code>decimal(P, S)</code></strong></td>
+<td><code>JSON string: "decimal(<P>,<S>)"</code></td>
+<td><code>"decimal(9,2)"</code>,<br /><code>"decimal(9, 2)"</code></td>
+</tr>
+<tr>
+<td><strong><code>struct</code></strong></td>
+<td><code>JSON object: {</code><br /> <code>"type": "struct",</code><br /> <code>"fields": [ {</code><br /> <code>"id": <field id int>,</code><br /> <code>"name": <name string>,</code><br /> <code>"required": <boolean>,</code><br /> <code>"type": <type JSON>,</code><br /> <code>"doc": <comment string></code><br /> &nbs [...]
+<td><code>{</code><br /> <code>"type": "struct",</code><br /> <code>"fields": [ {</code><br /> <code>"id": 1,</code><br /> <code>"name": "id",</code><br /> <code>"required": true,</code><br /> <code>"type": "uuid"</code><br /> <code>}, {</code><br /> <code>"id": 2,</code><br /> <code>"name": "data",</code><br /> &n [...]
+</tr>
+<tr>
+<td><strong><code>list</code></strong></td>
+<td><code>JSON object: {</code><br /> <code>"type": "list",</code><br /> <code>"element-id": <id int>,</code><br /> <code>"element-required": <bool></code><br /> <code>"element": <type JSON></code><br /><code>}</code></td>
+<td><code>{</code><br /> <code>"type": "list",</code><br /> <code>"element-id": 3,</code><br /> <code>"element-required": true,</code><br /> <code>"element": "string"</code><br /><code>}</code></td>
+</tr>
+<tr>
+<td><strong><code>map</code></strong></td>
+<td><code>JSON object: {</code><br /> <code>"type": "map",</code><br /> <code>"key-id": <key id int>,</code><br /> <code>"key": <type JSON>,</code><br /> <code>"value-id": <val id int>,</code><br /> <code>"value-required": <bool></code><br /> <code>"value": <type JSON></code><br /><code>}</code></td>
+<td><code>{</code><br /> <code>"type": "map",</code><br /> <code>"key-id": 4,</code><br /> <code>"key": "string",</code><br /> <code>"value-id": 5,</code><br /> <code>"value-required": false,</code><br /> <code>"value": "double"</code><br /><code>}</code></td>
+</tr>
+</tbody>
+</table>
+<h2 id="partition-specs">Partition Specs</h2>
+<p>Partition specs are serialized as a JSON object with the following fields:</p>
+<table>
+<thead>
+<tr>
+<th>Field</th>
+<th>JSON representation</th>
+<th>Example</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>spec-id</code></strong></td>
+<td><code>JSON int</code></td>
+<td><code>0</code></td>
+</tr>
+<tr>
+<td><strong><code>fields</code></strong></td>
+<td><code>JSON list: [</code><br /> <code><partition field JSON>,</code><br /> <code>...</code><br /><code>]</code></td>
+<td><code>[ {</code><br /> <code>"source-id": 4,</code><br /> <code>"name": "ts_day",</code><br /> <code>"transform": "day"</code><br /><code>}, {</code><br /> <code>"source-id": 1,</code><br /> <code>"name": "id_bucket",</code><br /> <code>"transform": "bucket[16]"</code><br /><code>} ]</code></td>
+</tr>
+</tbody>
+</table>
+<p>Each partition field in the fields list is stored as an object. See the table for more detail:</p>
+<table>
+<thead>
+<tr>
+<th>Transform or Field</th>
+<th>JSON representation</th>
+<th>Example</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>identity</code></strong></td>
+<td><code>JSON string: "identity"</code></td>
+<td><code>"identity"</code></td>
+</tr>
+<tr>
+<td><strong><code>bucket[N]</code></strong></td>
+<td><code>JSON string: "bucket<N>]"</code></td>
+<td><code>"bucket[16]"</code></td>
+</tr>
+<tr>
+<td><strong><code>truncate[W]</code></strong></td>
+<td><code>JSON string: "truncate[<W>]"</code></td>
+<td><code>"truncate[20]"</code></td>
+</tr>
+<tr>
+<td><strong><code>year</code></strong></td>
+<td><code>JSON string: "year"</code></td>
+<td><code>"year"</code></td>
+</tr>
+<tr>
+<td><strong><code>month</code></strong></td>
+<td><code>JSON string: "month"</code></td>
+<td><code>"month"</code></td>
+</tr>
+<tr>
+<td><strong><code>day</code></strong></td>
+<td><code>JSON string: "day"</code></td>
+<td><code>"day"</code></td>
+</tr>
+<tr>
+<td><strong><code>hour</code></strong></td>
+<td><code>JSON string: "hour"</code></td>
+<td><code>"hour"</code></td>
+</tr>
+<tr>
+<td><strong><code>Partition Field</code></strong></td>
+<td><code>JSON object: {</code><br /> <code>"source-id": <id int>,</code><br /> <code>"name": <name string>,</code><br /> <code>"transform": <transform JSON></code><br /><code>}</code></td>
+<td><code>{</code><br /> <code>"source-id": 1,</code><br /> <code>"name": "id_bucket",</code><br /> <code>"transform": "bucket[16]"</code><br /><code>}</code></td>
+</tr>
+</tbody>
+</table>
+<p>In some cases partition specs are stored using only the field list instead of the object format that includes the spec ID, like the deprecated <code>partition-spec</code> field in table metadata. The object format should be used unless otherwise noted in this spec.</p>
+<h2 id="table-metadata-and-snapshots">Table Metadata and Snapshots</h2>
+<p>Table metadata is serialized as a JSON object according to the following table. Snapshots are not serialized separately. Instead, they are stored in the table metadata JSON.</p>
+<table>
+<thead>
+<tr>
+<th>Metadata field</th>
+<th>JSON representation</th>
+<th>Example</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>format-version</code></strong></td>
+<td><code>JSON int</code></td>
+<td><code>1</code></td>
+</tr>
+<tr>
+<td><strong><code>location</code></strong></td>
+<td><code>JSON string</code></td>
+<td><code>"s3://b/wh/data.db/table"</code></td>
+</tr>
+<tr>
+<td><strong><code>last-updated-ms</code></strong></td>
+<td><code>JSON long</code></td>
+<td><code>1515100955770</code></td>
+</tr>
+<tr>
+<td><strong><code>last-column-id</code></strong></td>
+<td><code>JSON int</code></td>
+<td><code>22</code></td>
+</tr>
+<tr>
+<td><strong><code>schema</code></strong></td>
+<td><code>JSON schema (object)</code></td>
+<td><code>See above</code></td>
+</tr>
+<tr>
+<td><strong><code>partition-spec</code></strong></td>
+<td><code>JSON partition fields (list)</code></td>
+<td><code>See above, read partition-specs instead</code></td>
+</tr>
+<tr>
+<td><strong><code>partition-specs</code></strong></td>
+<td><code>JSON partition specs (list of objects)</code></td>
+<td><code>See above</code></td>
+</tr>
+<tr>
+<td><strong><code>default-spec-id</code></strong></td>
+<td><code>JSON int</code></td>
+<td><code>0</code></td>
+</tr>
+<tr>
+<td><strong><code>properties</code></strong></td>
+<td><code>JSON object: {</code><br /> <code>"<key>": "<val>",</code><br /> <code>...</code><br /><code>}</code></td>
+<td><code>{</code><br /> <code>"write.format.default": "avro",</code><br /> <code>"commit.retry.num-retries": "4"</code><br /><code>}</code></td>
+</tr>
+<tr>
+<td><strong><code>current-snapshot-id</code></strong></td>
+<td><code>JSON long</code></td>
+<td><code>3051729675574597004</code></td>
+</tr>
+<tr>
+<td><strong><code>snapshots</code></strong></td>
+<td><code>JSON list of objects: [ {</code><br /> <code>"snapshot-id": <id>,</code><br /> <code>"timestamp-ms": <timestamp-in-ms>,</code><br /> <code>"summary": {</code><br /> <code>"operation": <operation>,</code><br /> <code>... },</code><br /> <code>"manifest-list": "<location>"</code><br /> <code>},</code><br /> <code>...</code><br /><code>]</code></td>
+<td><code>[ {</code><br /> <code>"snapshot-id": 3051729675574597004,</code><br /> <code>"timestamp-ms": 1515100955770,</code><br /> <code>"summary": {</code><br /> <code>"operation": "append"</code><br /> <code>},</code><br /> <code>"manifest-list": "s3://b/wh/.../s1.avro"</code><br /><code>} ]</code></td>
+</tr>
+<tr>
+<td><strong><code>snapshot-log</code></strong></td>
+<td><code>JSON list of objects: [</code><br /> <code>{</code><br /> <code>"snapshot-id": ,</code><br /> <code>"timestamp-ms":</code><br /> <code>},</code><br /> <code>...</code><br /><code>]</code></td>
+<td><code>[ {</code><br /> <code>"snapshot-id": 30517296...,</code><br /> <code>"timestamp-ms": 1515100...</code><br /><code>} ]</code></td>
+</tr>
+</tbody>
+</table>
+<h1 id="appendix-d-single-value-serialization">Appendix D: Single-value serialization</h1>
+<p>This serialization scheme is for storing single values as individual binary values in the lower and upper bounds maps of manifest files.</p>
+<table>
+<thead>
+<tr>
+<th>Type</th>
+<th>Binary serialization</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><strong><code>boolean</code></strong></td>
+<td><code>0x00</code> for false, non-zero byte for true</td>
+</tr>
+<tr>
+<td><strong><code>int</code></strong></td>
+<td>Stored as 4-byte little-endian</td>
+</tr>
+<tr>
+<td><strong><code>long</code></strong></td>
+<td>Stored as 8-byte little-endian</td>
+</tr>
+<tr>
+<td><strong><code>float</code></strong></td>
+<td>Stored as 4-byte little-endian</td>
+</tr>
+<tr>
+<td><strong><code>double</code></strong></td>
+<td>Stored as 8-byte little-endian</td>
+</tr>
+<tr>
+<td><strong><code>date</code></strong></td>
+<td>Stores days from the 1970-01-01 in an 4-byte little-endian int</td>
+</tr>
+<tr>
+<td><strong><code>time</code></strong></td>
+<td>Stores microseconds from midnight in an 8-byte little-endian long</td>
+</tr>
+<tr>
+<td><strong><code>timestamp without zone</code></strong></td>
+<td>Stores microseconds from 1970-01-01 00:00:00.000000 in an 8-byte little-endian long</td>
+</tr>
+<tr>
+<td><strong><code>timestamp with zone</code></strong></td>
+<td>Stores microseconds from 1970-01-01 00:00:00.000000 UTC in an 8-byte little-endian long</td>
+</tr>
+<tr>
+<td><strong><code>string</code></strong></td>
+<td>UTF-8 bytes (without length)</td>
+</tr>
+<tr>
+<td><strong><code>uuid</code></strong></td>
+<td>16-byte big-endian value, see example in Appendix B</td>
+</tr>
+<tr>
+<td><strong><code>fixed(L)</code></strong></td>
+<td>Binary value</td>
+</tr>
+<tr>
+<td><strong><code>binary</code></strong></td>
+<td>Binary value (without length)</td>
+</tr>
+<tr>
+<td><strong><code>decimal(P, S)</code></strong></td>
+<td>Stores unscaled value as two’s-complement big-endian binary, using the minimum number of bytes for the value</td>
+</tr>
+<tr>
+<td><strong><code>struct</code></strong></td>
+<td>Not supported</td>
+</tr>
+<tr>
+<td><strong><code>list</code></strong></td>
+<td>Not supported</td>
+</tr>
+<tr>
+<td><strong><code>map</code></strong></td>
+<td>Not supported</td>
+</tr>
+</tbody>
+</table></div>
+
+
+ </div>
+
+ <footer class="col-md-12 text-center">
+
+ <hr>
+ <p>
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+
+
+ </footer>
+ <script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
+ <script src="../js/bootstrap-3.0.3.min.js"></script>
+ <script src="../js/highlight.pack.js"></script>
+ <script>hljs.initHighlightingOnLoad();</script>
+ <script>var base_url = ".."</script>
+
+ <script src="../js/base.js"></script>
+ <script src="../search/main.js"></script>
+
+ <div class="modal" id="mkdocs_search_modal" tabindex="-1" role="dialog" aria-labelledby="searchModalLabel" aria-hidden="true">
+ <div class="modal-dialog modal-lg">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h4 class="modal-title" id="searchModalLabel">Search</h4>
+ <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button>
+ </div>
+ <div class="modal-body">
+ <p>
+ From here you can search these documents. Enter
+ your search terms below.
+ </p>
+ <form>
+ <div class="form-group">
+ <input type="text" class="form-control" placeholder="Search..." id="mkdocs-search-query" title="Type search term here">
+ </div>
+ </form>
+ <div id="mkdocs-search-results"></div>
+ </div>
+ <div class="modal-footer">
+ </div>
+ </div>
+ </div>
+</div><div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
+ <div class="modal-dialog">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h4 class="modal-title" id="keyboardModalLabel">Keyboard Shortcuts</h4>
+ <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button>
+ </div>
+ <div class="modal-body">
+ <table class="table">
+ <thead>
+ <tr>
+ <th style="width: 20%;">Keys</th>
+ <th>Action</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td class="help shortcut"><kbd>?</kbd></td>
+ <td>Open this help</td>
+ </tr>
+ <tr>
+ <td class="next shortcut"><kbd>n</kbd></td>
+ <td>Next page</td>
+ </tr>
+ <tr>
+ <td class="prev shortcut"><kbd>p</kbd></td>
+ <td>Previous page</td>
+ </tr>
+ <tr>
+ <td class="search shortcut"><kbd>s</kbd></td>
+ <td>Search</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <div class="modal-footer">
+ </div>
+ </div>
+ </div>
+</div>
+ </body>
+
+</html>