You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/06/28 22:55:37 UTC
[iceberg] branch asf-site updated: Deployed a100d2d5a with MkDocs
version: 1.2.1
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 1ea887e Deployed a100d2d5a with MkDocs version: 1.2.1
1ea887e is described below
commit 1ea887e9f44702c2c2c6416692149159774e6b68
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Mon Jun 28 15:55:20 2021 -0700
Deployed a100d2d5a with MkDocs version: 1.2.1
---
404.html | 123 +-
api/index.html => api.html | 187 +-
aws/index.html => aws.html | 192 +-
blogs/index.html => blogs.html | 148 +-
community/index.html => community.html | 143 +-
configuration/index.html => configuration.html | 167 +-
css/base.css | 24 +-
css/base.min.css | 2 +-
css/cinder.css | 18 +-
css/cinder.min.css | 2 +-
custom-catalog/index.html => custom-catalog.html | 161 +-
evolution/index.html => evolution.html | 153 +-
flink/index.html => flink.html | 320 +-
getting-started/index.html => getting-started.html | 203 +-
hive.html | 973 ++
hive/index.html | 581 -
how-to-release/index.html => how-to-release.html | 176 +-
img/favicon.ico | Bin 1150 -> 1150 bytes
img/iceberg-metadata.png | Bin 15937 -> 140290 bytes
index.html | 135 +-
.../index.html => java-api-quickstart.html | 180 +-
.../apache/iceberg/flink/FlinkTableFactory.html | 2 +-
.../apache/iceberg/flink/FlinkTableFactory.html | 2 +-
.../jquery/jszip-utils/dist/jszip-utils-ie.js | 56 +
.../jquery/jszip-utils/dist/jszip-utils-ie.min.js | 10 +
.../0.9.0/jquery/jszip-utils/dist/jszip-utils.js | 118 +
.../jquery/jszip-utils/dist/jszip-utils.min.js | 10 +
javadoc/0.9.0/jquery/jszip/dist/jszip.js | 11623 +++++++++++++++++++
javadoc/0.9.0/jquery/jszip/dist/jszip.min.js | 15 +
js/base.js | 324 +-
maintenance/index.html => maintenance.html | 168 +-
nessie/index.html => nessie.html | 180 +-
partitioning/index.html => partitioning.html | 152 +-
performance/index.html => performance.html | 145 +-
presto/index.html | 11 -
.../index.html => python-api-intro.html | 197 +-
.../index.html => python-feature-support.html | 147 +-
.../index.html => python-quickstart.html | 147 +-
releases/index.html => releases.html | 145 +-
reliability/index.html => reliability.html | 145 +-
schemas/index.html => schemas.html | 145 +-
sitemap.xml | 240 +-
sitemap.xml.gz | Bin 230 -> 218 bytes
snapshots/index.html => snapshots.html | 139 +-
.../index.html => spark-configuration.html | 176 +-
spark-ddl/index.html => spark-ddl.html | 234 +-
.../index.html => spark-procedures.html | 197 +-
spark-queries/index.html => spark-queries.html | 226 +-
.../index.html => spark-structured-streaming.html | 155 +-
spark-writes/index.html => spark-writes.html | 221 +-
spark/index.html | 1226 --
spec/index.html => spec.html | 291 +-
terms/index.html => terms.html | 149 +-
time-travel/index.html | 11 -
trademarks/index.html => trademarks.html | 143 +-
trino/index.html => trino.html | 139 +-
index.html => why-iceberg.html | 202 +-
57 files changed, 16363 insertions(+), 5016 deletions(-)
diff --git a/404.html b/404.html
index 8fc0c3f..642b283 100644
--- a/404.html
+++ b/404.html
@@ -8,20 +8,26 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="/img/favicon.ico">
+ <link rel="shortcut icon" href="/img/favicon.ico">
<title>Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
<link href="/css/bootstrap-custom.min.css" rel="stylesheet">
<link href="/css/base.min.css" rel="stylesheet">
<link href="/css/cinder.min.css" rel="stylesheet">
- <link href="/css/highlight.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
<link href="/css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="/.">Apache Iceberg</a>
+
+ <a class="navbar-brand" href="/.">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="/.">About</a>
+ <a href="/index.html">About</a>
</li>
<li >
- <a href="/community/">Community</a>
+ <a href="/community.html">Community</a>
</li>
<li >
- <a href="/releases/">Releases</a>
+ <a href="/releases.html">Releases</a>
</li>
<li >
- <a href="/blogs/">Blogs</a>
+ <a href="/blogs.html">Blogs</a>
</li>
<li >
- <a href="/trademarks/">Trademarks</a>
+ <a href="/trademarks.html">Trademarks</a>
</li>
<li >
- <a href="/how-to-release/">How to Release</a>
+ <a href="/how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="/configuration/">Configuration</a>
+ <a href="/configuration.html">Configuration</a>
</li>
<li >
- <a href="/schemas/">Schemas</a>
+ <a href="/schemas.html">Schemas</a>
</li>
<li >
- <a href="/partitioning/">Partitioning</a>
+ <a href="/partitioning.html">Partitioning</a>
</li>
<li >
- <a href="/evolution/">Table evolution</a>
+ <a href="/evolution.html">Table evolution</a>
</li>
<li >
- <a href="/maintenance/">Maintenance</a>
+ <a href="/maintenance.html">Maintenance</a>
</li>
<li >
- <a href="/performance/">Performance</a>
+ <a href="/performance.html">Performance</a>
</li>
<li >
- <a href="/reliability/">Reliability</a>
+ <a href="/reliability.html">Reliability</a>
</li>
@@ -173,43 +172,43 @@
<li >
- <a href="/getting-started/">Getting Started</a>
+ <a href="/getting-started.html">Getting Started</a>
</li>
<li >
- <a href="/spark-configuration/">Configuration</a>
+ <a href="/spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="/spark-ddl/">DDL</a>
+ <a href="/spark-ddl.html">DDL</a>
</li>
<li >
- <a href="/spark-queries/">Queries</a>
+ <a href="/spark-queries.html">Queries</a>
</li>
<li >
- <a href="/spark-writes/">Writes</a>
+ <a href="/spark-writes.html">Writes</a>
</li>
<li >
- <a href="/spark-procedures/">Maintenance Procedures</a>
+ <a href="/spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="/spark-structured-streaming/">Structured Streaming</a>
+ <a href="/spark-structured-streaming.html">Structured Streaming</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="/flink/">Flink</a>
+ <a href="/flink.html">Flink</a>
</li>
<li >
- <a href="/hive/">Hive</a>
+ <a href="/hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="/aws/">AWS</a>
+ <a href="/aws.html">AWS</a>
</li>
<li >
- <a href="/nessie/">Nessie</a>
+ <a href="/nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="/api/">Java API intro</a>
+ <a href="/api.html">Java API intro</a>
</li>
<li >
- <a href="/java-api-quickstart/">Java Quickstart</a>
+ <a href="/java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="/custom-catalog/">Java Custom Catalog</a>
+ <a href="/custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="/python-quickstart/">Python Quickstart</a>
+ <a href="/python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="/python-api-intro/">Python API Intro</a>
+ <a href="/python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="/python-feature-support/">Python Feature Support</a>
+ <a href="/python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="/terms/">Definitions</a>
+ <a href="/terms.html">Definitions</a>
</li>
<li >
- <a href="/spec/">Spec</a>
+ <a href="/spec.html">Spec</a>
</li>
@@ -401,21 +400,31 @@
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script src="/js/bootstrap-3.0.3.min.js"></script>
- <script src="/js/highlight.pack.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
+
+
<script>var base_url = "/"</script>
<script src="/js/base.js"></script>
diff --git a/api/index.html b/api.html
similarity index 81%
rename from api/index.html
rename to api.html
index 722434d..3049c51 100644
--- a/api/index.html
+++ b/api.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Java API intro - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li class="active">
- <a href="./">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../nessie/">
+ <a rel="prev" href="nessie.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../java-api-quickstart/">
+ <a rel="next" href="java-api-quickstart.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -445,11 +444,11 @@
<p>The main purpose of the Iceberg API is to manage table metadata, like schema, partition spec, metadata, and data files that store table data.</p>
<p>Table metadata and operations are accessed through the <code>Table</code> interface. This interface will return table information.</p>
<h3 id="table-metadata">Table metadata<a class="headerlink" href="#table-metadata" title="Permanent link">¶</a></h3>
-<p>The <a href="../javadoc/master/?org/apache/iceberg/Table.html"><code>Table</code> interface</a> provides access to the table metadata:</p>
+<p>The <a href="javadoc/master/index.html?org/apache/iceberg/Table.html"><code>Table</code> interface</a> provides access to the table metadata:</p>
<ul>
-<li><code>schema</code> returns the current table <a href="../schemas/">schema</a></li>
+<li><code>schema</code> returns the current table <a href="schemas.html">schema</a></li>
<li><code>spec</code> returns the current table partition spec</li>
-<li><code>properties</code> returns a map of key-value <a href="../configuration/">properties</a></li>
+<li><code>properties</code> returns a map of key-value <a href="configuration.html">properties</a></li>
<li><code>currentSnapshot</code> returns the current table snapshot</li>
<li><code>snapshots</code> returns all valid snapshots for the table</li>
<li><code>snapshot(id)</code> returns a specific snapshot by ID</li>
@@ -462,32 +461,28 @@
</ul>
<h3 id="scanning">Scanning<a class="headerlink" href="#scanning" title="Permanent link">¶</a></h3>
<p>Iceberg table scans start by creating a <code>TableScan</code> object with <code>newScan</code>.</p>
-<pre><code class="java">TableScan scan = table.newScan();
+<pre><code class="language-java">TableScan scan = table.newScan();
</code></pre>
-
<p>To configure a scan, call <code>filter</code> and <code>select</code> on the <code>TableScan</code> to get a new <code>TableScan</code> with those changes.</p>
-<pre><code class="java">TableScan filteredScan = scan.filter(Expressions.equal("id", 5))
+<pre><code class="language-java">TableScan filteredScan = scan.filter(Expressions.equal("id", 5))
</code></pre>
-
<p>Calls to configuration methods create a new <code>TableScan</code> so that each <code>TableScan</code> is immutable and won’t change unexpectedly if shared across threads.</p>
<p>When a scan is configured, <code>planFiles</code>, <code>planTasks</code>, and <code>schema</code> are used to return files, tasks, and the read projection.</p>
-<pre><code class="java">TableScan scan = table.newScan()
+<pre><code class="language-java">TableScan scan = table.newScan()
.filter(Expressions.equal("id", 5))
.select("id", "data");
Schema projection = scan.schema();
Iterable<CombinedScanTask> tasks = scan.planTasks();
</code></pre>
-
<p>Use <code>asOfTime</code> or <code>useSnapshot</code> to configure the table snapshot for time travel queries.</p>
<h3 id="update-operations">Update operations<a class="headerlink" href="#update-operations" title="Permanent link">¶</a></h3>
-<p><code>Table</code> also exposes operations that update the table. These operations use a builder pattern, <a href="../javadoc/master/?org/apache/iceberg/PendingUpdate.html"><code>PendingUpdate</code></a>, that commits when <code>PendingUpdate#commit</code> is called.</p>
+<p><code>Table</code> also exposes operations that update the table. These operations use a builder pattern, <a href="javadoc/master/index.html?org/apache/iceberg/PendingUpdate.html"><code>PendingUpdate</code></a>, that commits when <code>PendingUpdate#commit</code> is called.</p>
<p>For example, updating the table schema is done by calling <code>updateSchema</code>, adding updates to the builder, and finally calling <code>commit</code> to commit the pending changes to the table:</p>
-<pre><code class="java">table.updateSchema()
+<pre><code class="language-java">table.updateSchema()
.addColumn("count", Types.LongType.get())
.commit();
</code></pre>
-
<p>Available operations to update a table are:</p>
<ul>
<li><code>updateSchema</code> – update the table schema</li>
@@ -505,7 +500,7 @@ Iterable<CombinedScanTask> tasks = scan.planTasks();
<h3 id="transactions">Transactions<a class="headerlink" href="#transactions" title="Permanent link">¶</a></h3>
<p>Transactions are used to commit multiple table changes in a single atomic operation. A transaction is used to create individual operations using factory methods, like <code>newAppend</code>, just like working with a <code>Table</code>. Operations created by a transaction are committed as a group when <code>commitTransaction</code> is called.</p>
<p>For example, deleting and appending a file in the same transaction:</p>
-<pre><code class="java">Transaction t = table.newTransaction();
+<pre><code class="language-java">Transaction t = table.newTransaction();
// commit operations to the transaction
t.newDelete().deleteFromRowFilter(filter).commit();
@@ -514,40 +509,35 @@ t.newAppend().appendFile(data).commit();
// commit all the changes to the table
t.commitTransaction();
</code></pre>
-
<h2 id="types">Types<a class="headerlink" href="#types" title="Permanent link">¶</a></h2>
-<p>Iceberg data types are located in the <a href="../javadoc/master/?org/apache/iceberg/types/package-summary.html"><code>org.apache.iceberg.types</code> package</a>.</p>
+<p>Iceberg data types are located in the <a href="javadoc/master/index.html?org/apache/iceberg/types/package-summary.html"><code>org.apache.iceberg.types</code> package</a>.</p>
<h3 id="primitives">Primitives<a class="headerlink" href="#primitives" title="Permanent link">¶</a></h3>
<p>Primitive type instances are available from static methods in each type class. Types without parameters use <code>get</code>, and types like <code>decimal</code> use factory methods:</p>
-<pre><code class="java">Types.IntegerType.get() // int
+<pre><code class="language-java">Types.IntegerType.get() // int
Types.DoubleType.get() // double
Types.DecimalType.of(9, 2) // decimal(9, 2)
</code></pre>
-
<h3 id="nested-types">Nested types<a class="headerlink" href="#nested-types" title="Permanent link">¶</a></h3>
<p>Structs, maps, and lists are created using factory methods in type classes.</p>
-<p>Like struct fields, map keys or values and list elements are tracked as nested fields. Nested fields track <a href="../evolution/#correctness">field IDs</a> and nullability.</p>
+<p>Like struct fields, map keys or values and list elements are tracked as nested fields. Nested fields track <a href="evolution.html#correctness">field IDs</a> and nullability.</p>
<p>Struct fields are created using <code>NestedField.optional</code> or <code>NestedField.required</code>. Map value and list element nullability is set in the map and list factory methods.</p>
-<pre><code class="java">// struct<1 id: int, 2 data: optional string>
+<pre><code class="language-java">// struct<1 id: int, 2 data: optional string>
StructType struct = Struct.of(
Types.NestedField.required(1, "id", Types.IntegerType.get()),
Types.NestedField.optional(2, "data", Types.StringType.get())
)
</code></pre>
-
-<pre><code class="java">// map<1 key: int, 2 value: optional string>
+<pre><code class="language-java">// map<1 key: int, 2 value: optional string>
MapType map = MapType.ofOptional(
1, Types.IntegerType.get(),
2, Types.StringType.get()
)
</code></pre>
-
-<pre><code class="java">// array<1 element: int>
+<pre><code class="language-java">// array<1 element: int>
ListType list = ListType.ofRequired(1, IntegerType.get());
</code></pre>
-
<h2 id="expressions">Expressions<a class="headerlink" href="#expressions" title="Permanent link">¶</a></h2>
-<p>Iceberg’s expressions are used to configure table scans. To create expressions, use the factory methods in <a href="../javadoc/master/?org/apache/iceberg/expressions/Expressions.html"><code>Expressions</code></a>.</p>
+<p>Iceberg’s expressions are used to configure table scans. To create expressions, use the factory methods in <a href="javadoc/master/index.html?org/apache/iceberg/expressions/Expressions.html"><code>Expressions</code></a>.</p>
<p>Supported predicate expressions are:</p>
<ul>
<li><code>isNull</code></li>
@@ -578,11 +568,10 @@ ListType list = ListType.ofRequired(1, IntegerType.get());
<p>For example, before using the expression <code>lessThan("x", 10)</code>, Iceberg needs to determine which column <code>"x"</code> refers to and convert <code>10</code> to that column’s data type.</p>
<p>If the expression could be bound to the type <code>struct<1 x: long, 2 y: long></code> or to <code>struct<11 x: int, 12 y: int></code>.</p>
<h3 id="expression-example">Expression example<a class="headerlink" href="#expression-example" title="Permanent link">¶</a></h3>
-<pre><code class="java">table.newScan()
+<pre><code class="language-java">table.newScan()
.filter(Expressions.greaterThanOrEqual("x", 5))
.filter(Expressions.lessThan("x", 10))
</code></pre>
-
<h2 id="modules">Modules<a class="headerlink" href="#modules" title="Permanent link">¶</a></h2>
<p>Iceberg table support is organized in library modules:</p>
<ul>
@@ -611,24 +600,34 @@ ListType list = ListType.ofRequired(1, IntegerType.get());
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/aws/index.html b/aws.html
similarity index 87%
rename from aws/index.html
rename to aws.html
index 6654d85..411caf2 100644
--- a/aws/index.html
+++ b/aws.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>AWS - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li class="active">
- <a href="./">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../hive/">
+ <a rel="prev" href="hive.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../nessie/">
+ <a rel="next" href="nessie.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -407,6 +406,7 @@
<li class="third-level"><a href="#spark">Spark</a></li>
<li class="third-level"><a href="#flink">Flink</a></li>
+ <li class="third-level"><a href="#hive">Hive</a></li>
<li class="second-level"><a href="#glue-catalog">Glue Catalog</a></li>
<li class="third-level"><a href="#glue-catalog-id">Glue Catalog ID</a></li>
@@ -469,7 +469,7 @@ you can go to the documentations of each engine to see how to load a custom cata
Here are some examples.</p>
<h3 id="spark">Spark<a class="headerlink" href="#spark" title="Permanent link">¶</a></h3>
<p>For example, to use AWS features with Spark 3 and AWS clients version 2.15.40, you can start the Spark SQL shell with:</p>
-<pre><code class="sh"># add Iceberg dependency
+<pre><code class="language-sh"># add Iceberg dependency
ICEBERG_VERSION=0.11.1
DEPENDENCIES="org.apache.iceberg:iceberg-spark3-runtime:$ICEBERG_VERSION"
@@ -493,11 +493,10 @@ spark-sql --packages $DEPENDENCIES \
--conf spark.sql.catalog.my_catalog.lock-impl=org.apache.iceberg.aws.glue.DynamoLockManager \
--conf spark.sql.catalog.my_catalog.lock.table=myGlueLockTable
</code></pre>
-
<p>As you can see, In the shell command, we use <code>--packages</code> to specify the additional AWS bundle and HTTP client dependencies with their version as <code>2.15.40</code>.</p>
<h3 id="flink">Flink<a class="headerlink" href="#flink" title="Permanent link">¶</a></h3>
<p>To use AWS module with Flink, you can download the necessary dependencies and specify them when starting the Flink SQL client:</p>
-<pre><code class="sh"># download Iceberg dependency
+<pre><code class="language-sh"># download Iceberg dependency
ICEBERG_VERSION=0.11.1
MAVEN_URL=https://repo1.maven.org/maven2
ICEBERG_MAVEN_URL=$MAVEN_URL/org/apache/iceberg
@@ -521,9 +520,8 @@ done
-j url-connection-client-$AWS_SDK_VERSION.jar \
shell
</code></pre>
-
<p>With those dependencies, you can create a Flink catalog like the following:</p>
-<pre><code class="sql">CREATE CATALOG my_catalog WITH (
+<pre><code class="language-sql">CREATE CATALOG my_catalog WITH (
'type'='iceberg',
'warehouse'='s3://my-bucket/my/key/prefix',
'catalog-impl'='org.apache.iceberg.aws.glue.GlueCatalog',
@@ -532,9 +530,8 @@ done
'lock.table'='myGlueLockTable'
);
</code></pre>
-
<p>You can also specify the catalog configurations in <code>sql-client-defaults.yaml</code> to preload it:</p>
-<pre><code class="yaml">catalogs:
+<pre><code class="language-yaml">catalogs:
- name: my_catalog
type: iceberg
warehouse: s3://my-bucket/my/key/prefix
@@ -543,7 +540,27 @@ done
lock-impl: org.apache.iceberg.aws.glue.DynamoLockManager
lock.table: myGlueLockTable
</code></pre>
-
+<h3 id="hive">Hive<a class="headerlink" href="#hive" title="Permanent link">¶</a></h3>
+<p>To use AWS module with Hive, you can download the necessary dependencies similar to the Flink example,
+and then add them to the Hive classpath or add the jars at runtime in CLI:</p>
+<pre><code>add jar /my/path/to/iceberg-hive-runtime.jar;
+add jar /my/path/to/aws/bundle.jar;
+add jar /my/path/to/aws/url-connection-client.jar;
+</code></pre>
+<p>With those dependencies, you can register a Glue catalog and create external tables in Hive at runtime in CLI by:</p>
+<pre><code class="language-sql">SET iceberg.engine.hive.enabled=true;
+SET hive.vectorized.execution.enabled=false;
+SET iceberg.catalog.glue.catalog-impl=org.apache.iceberg.aws.glue.GlueCatalog;
+SET iceberg.catalog.glue.warehouse=s3://my-bucket/my/key/prefix;
+SET iceberg.catalog.glue.lock-impl=org.apache.iceberg.aws.glue.DynamoLockManager;
+SET iceberg.catalog.glue.lock.table=myGlueLockTable;
+
+-- suppose you have an Iceberg table database_a.table_a created by GlueCatalog
+CREATE EXTERNAL TABLE database_a.table_a
+STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
+TBLPROPERTIES ('iceberg.catalog'='glue');
+</code></pre>
+<p>You can also preload the catalog by setting the configurations above in <code>hive-site.xml</code>.</p>
<h2 id="glue-catalog">Glue Catalog<a class="headerlink" href="#glue-catalog" title="Permanent link">¶</a></h2>
<p>Iceberg enables the use of <a href="https://aws.amazon.com/glue">AWS Glue</a> as the <code>Catalog</code> implementation.
When used, an Iceberg namespace is stored as a <a href="https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-databases.html">Glue Database</a>,
@@ -591,7 +608,7 @@ then any newly created table will have a default root location under the new pre
For instance, a new table <code>my_table_2</code> will have its root location at <code>s3://my-ns-bucket/my_table_2</code>.</p>
<p>To use a completely different root path for a specific table, set the <code>location</code> table property to the desired root path value you want.
For example, in Spark SQL you can do:</p>
-<pre><code class="sql">CREATE TABLE my_catalog.my_ns.my_table (
+<pre><code class="language-sql">CREATE TABLE my_catalog.my_ns.my_table (
id bigint,
data string,
category string)
@@ -599,9 +616,8 @@ USING iceberg
OPTIONS ('location'='s3://my-special-table-bucket')
PARTITIONED BY (category);
</code></pre>
-
<p>For engines like Spark that supports the <code>LOCATION</code> keyword, the above SQL statement is equivalent to:</p>
-<pre><code class="sql">CREATE TABLE my_catalog.my_ns.my_table (
+<pre><code class="language-sql">CREATE TABLE my_catalog.my_ns.my_table (
id bigint,
data string,
category string)
@@ -609,7 +625,6 @@ USING iceberg
LOCATION 's3://my-special-table-bucket'
PARTITIONED BY (category);
</code></pre>
-
<h2 id="s3-fileio">S3 FileIO<a class="headerlink" href="#s3-fileio" title="Permanent link">¶</a></h2>
<p>Iceberg allows users to write data to S3 through <code>S3FileIO</code>.
<code>GlueCatalog</code> by default uses this <code>FileIO</code>, and other catalogs can load this <code>FileIO</code> using the <code>io-impl</code> catalog property.</p>
@@ -695,7 +710,7 @@ Iceberg by default uses the Hive storage layout, but can be switched to use a di
In this mode, a hash string is added to the beginning of each file path, so that files are equally distributed across all prefixes in an S3 bucket.
This results in minimized throttling and maximized throughput for S3-related IO operations.
Here is an example Spark SQL command to create a table with this feature enabled:</p>
-<pre><code class="sql">CREATE TABLE my_catalog.my_ns.my_table (
+<pre><code class="language-sql">CREATE TABLE my_catalog.my_ns.my_table (
id bigint,
data string,
category string)
@@ -705,7 +720,6 @@ OPTIONS (
'write.object-storage.path'='s3://my-table-data-bucket')
PARTITIONED BY (category);
</code></pre>
-
<p>For more details, please refer to the <a href="../custom-catalog/#custom-location-provider-implementation">LocationProvider Configuration</a> section. </p>
<h3 id="s3-strong-consistency">S3 Strong Consistency<a class="headerlink" href="#s3-strong-consistency" title="Permanent link">¶</a></h3>
<p>In November 2020, S3 announced <a href="https://aws.amazon.com/s3/consistency/">strong consistency</a> for all read operations, and Iceberg is updated to fully leverage this feature.
@@ -767,7 +781,7 @@ This also serves as an example for users who would like to implement their own A
<p>By using this client factory, an STS client is initialized with the default credential and region to assume the specified role.
The Glue, S3 and DynamoDB clients are then initialized with the assume-role credential and region to access resources.
Here is an example to start Spark shell with this client factory:</p>
-<pre><code class="shell">spark-sql --packages org.apache.iceberg:iceberg-spark3-runtime:0.11.1,software.amazon.awssdk:bundle:2.15.40 \
+<pre><code class="language-shell">spark-sql --packages org.apache.iceberg:iceberg-spark3-runtime:0.11.1,software.amazon.awssdk:bundle:2.15.40 \
--conf spark.sql.catalog.my_catalog=org.apache.iceberg.spark.SparkCatalog \
--conf spark.sql.catalog.my_catalog.warehouse=s3://my-bucket/my/key/prefix \
--conf spark.sql.catalog.my_catalog.catalog-impl=org.apache.iceberg.aws.glue.GlueCatalog \
@@ -775,14 +789,13 @@ Here is an example to start Spark shell with this client factory:</p>
--conf spark.sql.catalog.my_catalog.client.assume-role.arn=arn:aws:iam::123456789:role/myRoleToAssume \
--conf spark.sql.catalog.my_catalog.client.assume-role.region=ap-northeast-1
</code></pre>
-
<h2 id="run-iceberg-on-aws">Run Iceberg on AWS<a class="headerlink" href="#run-iceberg-on-aws" title="Permanent link">¶</a></h2>
<h3 id="amazon-emr">Amazon EMR<a class="headerlink" href="#amazon-emr" title="Permanent link">¶</a></h3>
<p><a href="https://aws.amazon.com/emr/">Amazon EMR</a> can provision clusters with <a href="https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-spark.html">Spark</a> (EMR 6 for Spark 3, EMR 5 for Spark 2),
<a href="https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-hive.html">Hive</a>, <a href="https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-flink.html">Flink</a>,
<a href="https://docs.aws.amazon.com/emr/latest/ReleaseGuide/emr-presto.html">Trino</a> that can run Iceberg.</p>
<p>You can use a <a href="https://docs.aws.amazon.com/emr/latest/ManagementGuide/emr-plan-bootstrap.html">bootstrap action</a> similar to the following to pre-install all necessary dependencies:</p>
-<pre><code class="sh">#!/bin/bash
+<pre><code class="language-sh">#!/bin/bash
AWS_SDK_VERSION=2.15.40
ICEBERG_VERSION=0.11.1
@@ -817,7 +830,6 @@ install_dependencies () {
install_dependencies $LIB_PATH $ICEBERG_MAVEN_URL $ICEBERG_VERSION "${ICEBERG_PACKAGES[@]}"
install_dependencies $LIB_PATH $AWS_MAVEN_URL $AWS_SDK_VERSION "${AWS_PACKAGES[@]}"
</code></pre>
-
<h3 id="amazon-kinesis">Amazon Kinesis<a class="headerlink" href="#amazon-kinesis" title="Permanent link">¶</a></h3>
<p><a href="https://aws.amazon.com/about-aws/whats-new/2019/11/you-can-now-run-fully-managed-apache-flink-applications-with-apache-kafka/">Amazon Kinesis Data Analytics</a> provides a platform
to run fully managed Apache Flink applications. You can include Iceberg in your application Jar and run it in the platform.</p></div>
@@ -825,24 +837,34 @@ to run fully managed Apache Flink applications. You can include Iceberg in your
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/blogs/index.html b/blogs.html
similarity index 79%
rename from blogs/index.html
rename to blogs.html
index 2b93865..424168f 100644
--- a/blogs/index.html
+++ b/blogs.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Blogs - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li class="active">
- <a href="./">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../releases/">
+ <a rel="prev" href="releases.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../trademarks/">
+ <a rel="next" href="trademarks.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -403,6 +402,8 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
<ul class="nav bs-sidenav">
<li class="first-level active"><a href="#iceberg-blogs">Iceberg Blogs</a></li>
+ <li class="second-level"><a href="#migrating-to-apache-iceberg-at-adobe-experience-platform">Migrating to Apache Iceberg at Adobe Experience Platform</a></li>
+
<li class="second-level"><a href="#a-short-introduction-to-apache-iceberg">A Short Introduction to Apache Iceberg</a></li>
<li class="second-level"><a href="#taking-query-optimizations-to-the-next-level-with-iceberg">Taking Query Optimizations to the Next Level with Iceberg</a></li>
@@ -440,6 +441,9 @@
<h2 id="iceberg-blogs">Iceberg Blogs<a class="headerlink" href="#iceberg-blogs" title="Permanent link">¶</a></h2>
<p>Here is a list of company blogs that talk about Iceberg. The blogs are ordered from most recent to oldest.</p>
+<h3 id="migrating-to-apache-iceberg-at-adobe-experience-platform"><a href="https://medium.com/adobetech/migrating-to-apache-iceberg-at-adobe-experience-platform-40fa80f8b8de">Migrating to Apache Iceberg at Adobe Experience Platform</a><a class="headerlink" href="#migrating-to-apache-iceberg-at-adobe-experience-platform" title="Permanent link">¶</a></h3>
+<p><strong>Date</strong>: Jun 17th, 2021, <strong>Company</strong>: Adobe</p>
+<p><strong>Author</strong>: <a href="https://www.linkedin.com/in/rominparekh/">Romin Parekh</a>, <a href="https://www.linkedin.com/in/miao-wang-0406a74/">Miao Wang</a>, <a href="https://www.linkedin.com/in/shonesadler/">Shone Sadler</a></p>
<h3 id="a-short-introduction-to-apache-iceberg"><a href="https://medium.com/expedia-group-tech/a-short-introduction-to-apache-iceberg-d34f628b6799">A Short Introduction to Apache Iceberg</a><a class="headerlink" href="#a-short-introduction-to-apache-iceberg" title="Permanent link">¶</a></h3>
<p><strong>Date</strong>: Jan 26th, 2021, <strong>Company</strong>: Expedia</p>
<p><strong>Author</strong>: <a href="https://www.linkedin.com/in/christine-mathiesen-676a98159/">Christine Mathiesen</a></p>
@@ -465,24 +469,34 @@
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/community/index.html b/community.html
similarity index 82%
rename from community/index.html
rename to community.html
index 1a11823..32b8c3f 100644
--- a/community/index.html
+++ b/community.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Community - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li class="active">
- <a href="./">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="..">
+ <a rel="prev" href="index.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../releases/">
+ <a rel="next" href="releases.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -482,24 +481,34 @@
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/configuration/index.html b/configuration.html
similarity index 84%
rename from configuration/index.html
rename to configuration.html
index e4059b4..623438f 100644
--- a/configuration/index.html
+++ b/configuration.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Configuration - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li class="active">
- <a href="./">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../how-to-release/">
+ <a rel="prev" href="how-to-release.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../schemas/">
+ <a rel="next" href="schemas.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -537,7 +536,7 @@
</tr>
<tr>
<td>write.target-file-size-bytes</td>
-<td>Long.MAX_VALUE</td>
+<td>536870912 (512 MB)</td>
<td>Controls the size of files generated to target about this many bytes</td>
</tr>
<tr>
@@ -603,6 +602,26 @@
<td>Maximum time in milliseconds to wait before retrying a commit</td>
</tr>
<tr>
+<td>commit.status-check.num-retries</td>
+<td>3</td>
+<td>Number of times to check whether a commit succeeded after a connection is lost before failing due to an unknown commit state</td>
+</tr>
+<tr>
+<td>commit.status-check.min-wait-ms</td>
+<td>1000 (1s)</td>
+<td>Minimum time in milliseconds to wait before retrying a status-check</td>
+</tr>
+<tr>
+<td>commit.status-check.max-wait-ms</td>
+<td>60000 (1 min)</td>
+<td>Maximum time in milliseconds to wait before retrying a status-check</td>
+</tr>
+<tr>
+<td>commit.status-check.total-timeout-ms</td>
+<td>1800000 (30 min)</td>
+<td>Maximum time in milliseconds to wait before retrying a status-check</td>
+</tr>
+<tr>
<td>commit.manifest.target-size-bytes</td>
<td>8388608 (8 MB)</td>
<td>Target size when merging manifest files</td>
@@ -687,7 +706,7 @@
<p><code>HadoopCatalog</code> and <code>HiveCatalog</code> can access the properties in their constructors.
Any other custom catalog can access the properties by implementing <code>Catalog.initialize(catalogName, catalogProperties)</code>.
The properties can be manually constructed or passed in from a compute engine like Spark or Flink.
-Spark uses its session properties as catalog properties, see more details in the <a href="../spark-configuration/#catalog-configuration">Spark configuration</a> section.
+Spark uses its session properties as catalog properties, see more details in the <a href="spark-configuration.html#catalog-configuration">Spark configuration</a> section.
Flink passes in catalog properties through <code>CREATE CATALOG</code> statement, see more details in the <a href="../flink/#creating-catalogs-and-using-catalogs">Flink</a> section.</p>
<h3 id="lock-catalog-properties">Lock catalog properties<a class="headerlink" href="#lock-catalog-properties" title="Permanent link">¶</a></h3>
<p>Here are the catalog properties related to locking. They are used by some catalog implementations to control the locking behavior during commits.</p>
@@ -772,24 +791,34 @@ Hive Metastore before the lock is retried from Iceberg.</p></div>
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/css/base.css b/css/base.css
index 7b45cbb..2d68482 100644
--- a/css/base.css
+++ b/css/base.css
@@ -1,12 +1,9 @@
-body {
- padding-top: 70px;
+html {
+ scroll-padding-top: 70px;
}
-h1[id]:before, h2[id]:before, h3[id]:before, h4[id]:before, h5[id]:before, h6[id]:before {
- content: "";
- display: block;
- margin-top: -75px;
- height: 75px;
+body {
+ padding-top: 70px;
}
p > img {
@@ -103,7 +100,8 @@ div.source-links {
.bs-sidebar.affix {
position: fixed; /* Undo the static from mobile first approach */
top: 80px;
- max-height: calc(100% - 90px);
+ max-height: calc(100% - 180px);
+ overflow-y: auto;
}
.bs-sidebar.affix-bottom {
position: absolute; /* Undo the static from mobile first approach */
@@ -228,6 +226,16 @@ div.source-links {
}
/* End Bootstrap Callouts CSS Source by Chris Pratt */
+/* Headerlinks */
+.headerlink {
+ display: none;
+ padding-left: .5em;
+}
+
+h1:hover .headerlink, h2:hover .headerlink, h3:hover .headerlink, h4:hover .headerlink, h5:hover .headerlink, h6:hover .headerlink {
+ display: inline-block;
+}
+
/* Admonitions */
.admonition {
padding: 20px;
diff --git a/css/base.min.css b/css/base.min.css
index c5454f8..f7e580f 100644
--- a/css/base.min.css
+++ b/css/base.min.css
@@ -1 +1 @@
-body{padding-top:70px}h1[id]:before,h2[id]:before,h3[id]:before,h4[id]:before,h5[id]:before,h6[id]:before{content:"";display:block;margin-top:-75px;height:75px}p>img{max-width:100%;height:auto}ul.nav li.first-level{font-weight:bold}ul.nav li.third-level{padding-left:12px}div.col-md-3{padding-left:0}div.col-md-9{padding-bottom:100px}div.source-links{float:right}.bs-sidebar.affix{position:static}.bs-sidebar.well{padding:0}.bs-sidenav{margin-top:30px;margin-bottom:30px;padding-top:10px;padd [...]
+html{scroll-padding-top:70px}body{padding-top:70px}p>img{max-width:100%;height:auto}ul.nav li.first-level{font-weight:bold}ul.nav li.third-level{padding-left:12px}div.col-md-3{padding-left:0}div.col-md-9{padding-bottom:100px}div.source-links{float:right}.bs-sidebar.affix{position:static}.bs-sidebar.well{padding:0}.bs-sidenav{margin-top:30px;margin-bottom:30px;padding-top:10px;padding-bottom:10px;border-radius:5px}.bs-sidebar .nav>li>a{display:block;padding:5px 20px;z-index:1}.bs-sidebar [...]
diff --git a/css/cinder.css b/css/cinder.css
index ce3cb62..4834560 100644
--- a/css/cinder.css
+++ b/css/cinder.css
@@ -10,23 +10,28 @@ body {
color: #343838;
}
h1, h2, h3, h4, h5, h6 {
- font-family:'PT Sans', 'Helvetica Neue', Helvetica, Arial, sans-serif;
+ font-family:'Inter', 'Helvetica Neue', Helvetica, Arial, sans-serif;
color: #222;
}
h1 small, h2 small, h3 small, h4 small, h5 small, h6 small, .h1 small, .h2 small, .h3 small, .h4 small, .h5 small, .h6 small, h1 .small, h2 .small, h3 .small, h4 .small, h5 .small, h6 .small, .h1 .small, .h2 .small, .h3 .small, .h4 .small, .h5 .small, .h6 .small {
color: #B1B7B9;
}
+
+h2 {
+ margin-top: 35px;
+}
+
h1, h2 {
font-weight: 700;
}
h4 {
- font-family: 'Open Sans', 'Helvetica Neue', Helvetica, Arial, sans-serif;
+ font-family: 'Inter', 'Helvetica Neue', Helvetica, Arial, sans-serif;
font-weight: 300;
margin-top: 20px;
font-style: italic;
}
h5 {
- font-family: 'Open Sans', 'Helvetica Neue', Helvetica, Arial, sans-serif;
+ font-family: 'Inter', 'Helvetica Neue', Helvetica, Arial, sans-serif;
font-weight: 300;
font-variant: small-caps;
}
@@ -41,7 +46,7 @@ pre {
margin-bottom: 25px;
}
.lead {
- font-family:"Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-family:"Inter", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-weight: 400;
line-height: 1.4;
letter-spacing: 0.0312em;
@@ -54,7 +59,8 @@ pre {
.bs-sidenav {
background-image: url("../img/grid11.png");
background-repeat: repeat;
- font-size: 12px;
+ font-family: Inter,"Helvetica Neue",Helvetica,Arial,sans-serif;
+ font-size: 13px;
}
.well {
background-color: #FCFDFF;
@@ -71,7 +77,7 @@ pre {
box-shadow: none;
}
#mkdocs-search-query {
- font-family:"Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
+ font-family:"Inter", "Helvetica Neue", Helvetica, Arial, sans-serif;
font-size: 20px;
font-weight: 700;
color: #343838;
diff --git a/css/cinder.min.css b/css/cinder.min.css
index ff25572..fb7cc34 100644
--- a/css/cinder.min.css
+++ b/css/cinder.min.css
@@ -1 +1 @@
-body{font-family:"Open Sans","Helvetica Neue",Helvetica,Arial,sans-serif;font-size:16px;line-height:1.7;background-color:#FFF;color:#343838}h1,h2,h3,h4,h5,h6{font-family:'PT Sans','Helvetica Neue',Helvetica,Arial,sans-serif;color:#222}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small,.h1 small,.h2 small,.h3 small,.h4 small,.h5 small,.h6 small,h1 .small,h2 .small,h3 .small,h4 .small,h5 .small,h6 .small,.h1 .small,.h2 .small,.h3 .small,.h4 .small,.h5 .small,.h6 .small{color:#b1b7b9}h1, [...]
+body{font-family:"Open Sans","Helvetica Neue",Helvetica,Arial,sans-serif;font-size:16px;line-height:1.7;background-color:#FFF;color:#343838}h1,h2,h3,h4,h5,h6{font-family:'Inter','Helvetica Neue',Helvetica,Arial,sans-serif;color:#222}h1 small,h2 small,h3 small,h4 small,h5 small,h6 small,.h1 small,.h2 small,.h3 small,.h4 small,.h5 small,.h6 small,h1 .small,h2 .small,h3 .small,h4 .small,h5 .small,h6 .small,.h1 .small,.h2 .small,.h3 .small,.h4 .small,.h5 .small,.h6 .small{color:#b1b7b9}h2{ma [...]
diff --git a/custom-catalog/index.html b/custom-catalog.html
similarity index 84%
rename from custom-catalog/index.html
rename to custom-catalog.html
index b689fe5..264906e 100644
--- a/custom-catalog/index.html
+++ b/custom-catalog.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Java Custom Catalog - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li class="active">
- <a href="./">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../java-api-quickstart/">
+ <a rel="prev" href="java-api-quickstart.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../python-quickstart/">
+ <a rel="next" href="python-quickstart.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -446,7 +445,7 @@
<h3 id="custom-table-operations-implementation">Custom table operations implementation<a class="headerlink" href="#custom-table-operations-implementation" title="Permanent link">¶</a></h3>
<p>Extend <code>BaseMetastoreTableOperations</code> to provide implementation on how to read and write metadata</p>
<p>Example:</p>
-<pre><code class="java">class CustomTableOperations extends BaseMetastoreTableOperations {
+<pre><code class="language-java">class CustomTableOperations extends BaseMetastoreTableOperations {
private String dbName;
private String tableName;
private Configuration conf;
@@ -493,13 +492,12 @@
}
}
</code></pre>
-
<p>A <code>TableOperations</code> instance is usually obtained by calling <code>Catalog.newTableOps(TableIdentifier)</code>.
See the next section about implementing and loading a custom catalog.</p>
<h3 id="custom-catalog-implementation_1">Custom catalog implementation<a class="headerlink" href="#custom-catalog-implementation_1" title="Permanent link">¶</a></h3>
<p>Extend <code>BaseMetastoreCatalog</code> to provide default warehouse locations and instantiate <code>CustomTableOperations</code></p>
<p>Example:</p>
-<pre><code class="java">public class CustomCatalog extends BaseMetastoreCatalog {
+<pre><code class="language-java">public class CustomCatalog extends BaseMetastoreCatalog {
private Configuration configuration;
@@ -556,7 +554,6 @@ See the next section about implementing and loading a custom catalog.</p>
}
}
</code></pre>
-
<p>Catalog implementations can be dynamically loaded in most compute engines.
For Spark and Flink, you can specify the <code>catalog-impl</code> catalog property to load it.
Read the <a href="../configuration/#catalog-properties">Configuration</a> section for more details.
@@ -565,7 +562,7 @@ If your catalog must read Hadoop configuration to access certain environment pro
<h3 id="custom-file-io-implementation">Custom file IO implementation<a class="headerlink" href="#custom-file-io-implementation" title="Permanent link">¶</a></h3>
<p>Extend <code>FileIO</code> and provide implementation to read and write data files</p>
<p>Example:</p>
-<pre><code class="java">public class CustomFileIO implements FileIO {
+<pre><code class="language-java">public class CustomFileIO implements FileIO {
// must have a no-arg constructor to be dynamically loaded
// initialize(Map<String, String> properties) will be called to complete initialization
@@ -600,7 +597,6 @@ If your catalog must read Hadoop configuration to access certain environment pro
}
}
</code></pre>
-
<p>If you are already implementing your own catalog, you can implement <code>TableOperations.io()</code> to use your custom <code>FileIO</code>.
In addition, custom <code>FileIO</code> implementations can also be dynamically loaded in <code>HadoopCatalog</code> and <code>HiveCatalog</code> by specifying the <code>io-impl</code> catalog property.
Read the <a href="../configuration/#catalog-properties">Configuration</a> section for more details.
@@ -608,7 +604,7 @@ If your <code>FileIO</code> must read Hadoop configuration to access certain env
<h3 id="custom-location-provider-implementation">Custom location provider implementation<a class="headerlink" href="#custom-location-provider-implementation" title="Permanent link">¶</a></h3>
<p>Extend <code>LocationProvider</code> and provide implementation to determine the file path to write data</p>
<p>Example:</p>
-<pre><code class="java">public class CustomLocationProvider implements LocationProvider {
+<pre><code class="language-java">public class CustomLocationProvider implements LocationProvider {
private String tableLocation;
@@ -630,11 +626,10 @@ If your <code>FileIO</code> must read Hadoop configuration to access certain env
}
}
</code></pre>
-
<p>If you are already implementing your own catalog, you can override <code>TableOperations.locationProvider()</code> to use your custom default <code>LocationProvider</code>.
To use a different custom location provider for a specific table, specify the implementation when creating the table using table property <code>write.location-provider.impl</code></p>
<p>Example:</p>
-<pre><code class="sql">CREATE TABLE hive.default.my_table (
+<pre><code class="language-sql">CREATE TABLE hive.default.my_table (
id bigint,
data string,
category string)
@@ -644,11 +639,10 @@ OPTIONS (
)
PARTITIONED BY (category);
</code></pre>
-
<h3 id="custom-icebergsource">Custom IcebergSource<a class="headerlink" href="#custom-icebergsource" title="Permanent link">¶</a></h3>
<p>Extend <code>IcebergSource</code> and provide implementation to read from <code>CustomCatalog</code></p>
<p>Example:</p>
-<pre><code class="java">public class CustomIcebergSource extends IcebergSource {
+<pre><code class="language-java">public class CustomIcebergSource extends IcebergSource {
@Override
protected Table findTable(DataSourceOptions options, Configuration conf) {
@@ -662,30 +656,39 @@ PARTITIONED BY (category);
}
}
</code></pre>
-
<p>Register the <code>CustomIcebergSource</code> by updating <code>META-INF/services/org.apache.spark.sql.sources.DataSourceRegister</code> with its fully qualified name</p></div>
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/evolution/index.html b/evolution.html
similarity index 81%
rename from evolution/index.html
rename to evolution.html
index 91d2357..588dc46 100644
--- a/evolution/index.html
+++ b/evolution.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Table evolution - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li class="active">
- <a href="./">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../partitioning/">
+ <a rel="prev" href="partitioning.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../maintenance/">
+ <a rel="next" href="maintenance.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -461,19 +460,18 @@
<h2 id="partition-evolution">Partition evolution<a class="headerlink" href="#partition-evolution" title="Permanent link">¶</a></h2>
<p>Iceberg table partitioning can be updated in an existing table because queries do not reference partition values directly.</p>
<p>When you evolve a partition spec, the old data written with an earlier spec remains unchanged. New data is written using the new spec in a new layout. Metadata for each of the partition versions is kept separately. Because of this, when you start writing queries, you get split planning. This is where each partition layout plans files separately using the filter it derives for that specific partition layout. Here’s a visual representation of a contrived example: </p>
-<p><img alt="Partition evolution diagram" src="../img/partition-spec-evolution.png" />
+<p><img alt="Partition evolution diagram" src="img/partition-spec-evolution.png" />
<em>The data for 2008 is partitioned by month. Starting from 2009 the table is updated so that the data is instead partitioned by day. Both partitioning layouts are able to coexist in the same table.</em></p>
-<p>Iceberg uses <a href="../partitioning/">hidden partitioning</a>, so you don’t <em>need</em> to write queries for a specific partition layout to be fast. Instead, you can write queries that select the data you need, and Iceberg automatically prunes out files that don’t contain matching data.</p>
+<p>Iceberg uses <a href="partitioning.html">hidden partitioning</a>, so you don’t <em>need</em> to write queries for a specific partition layout to be fast. Instead, you can write queries that select the data you need, and Iceberg automatically prunes out files that don’t contain matching data.</p>
<p>Partition evolution is a metadata operation and does not eagerly rewrite files.</p>
<p>Iceberg’s Java table API provides <code>updateSpec</code> API to update partition spec.
For example, the following code could be used to update the partition spec to add a new partition field that places <code>id</code> column values into 8 buckets and remove an existing partition field <code>category</code>:</p>
-<pre><code class="java">Table sampleTable = ...;
+<pre><code class="language-java">Table sampleTable = ...;
sampleTable.updateSpec()
.addField(bucket("id", 8))
.removeField("category")
.commit();
</code></pre>
-
<p>Spark supports updating partition spec through its <code>ALTER TABLE</code> SQL statement, see more details in <a href="../spark/#alter-table-add-partition-field">Spark SQL</a>.</p>
<h2 id="sort-order-evolution">Sort order evolution<a class="headerlink" href="#sort-order-evolution" title="Permanent link">¶</a></h2>
<p>Similar to partition spec, Iceberg sort order can also be updated in an existing table.
@@ -483,36 +481,45 @@ Engines can always choose to write data in the latest sort order or unsorted whe
For example, the following code could be used to create a new sort order
with <code>id</code> column sorted in ascending order with nulls last,
and <code>category</code> column sorted in descending order with nulls first:</p>
-<pre><code class="java">Table sampleTable = ...;
+<pre><code class="language-java">Table sampleTable = ...;
sampleTable.replaceSortOrder()
.asc("id", NullOrder.NULLS_LAST)
.dec("category", NullOrder.NULL_FIRST)
.commit();
</code></pre>
-
<p>Spark supports updating sort order through its <code>ALTER TABLE</code> SQL statement, see more details in <a href="../spark/#alter-table-write-ordered-by">Spark SQL</a>.</p></div>
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/flink/index.html b/flink.html
similarity index 74%
rename from flink/index.html
rename to flink.html
index 8427b2e..4e87661 100644
--- a/flink/index.html
+++ b/flink.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Flink - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li class="active">
- <a href="./">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../spark-structured-streaming/">
+ <a rel="prev" href="spark-structured-streaming.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../hive/">
+ <a rel="next" href="hive.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -403,7 +402,9 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
<ul class="nav bs-sidenav">
<li class="first-level active"><a href="#flink">Flink</a></li>
- <li class="second-level"><a href="#preparation">Preparation</a></li>
+ <li class="second-level"><a href="#preparation-when-using-flink-sql-client">Preparation when using Flink SQL Client</a></li>
+
+ <li class="second-level"><a href="#preparation-when-using-flinks-python-api">Preparation when using Flink's Python API</a></li>
<li class="second-level"><a href="#creating-catalogs-and-using-catalogs">Creating catalogs and using catalogs.</a></li>
@@ -464,8 +465,8 @@
-->
<h1 id="flink">Flink<a class="headerlink" href="#flink" title="Permanent link">¶</a></h1>
-<p>Apache Iceberg support both <a href="https://flink.apache.org/">Apache Flink</a>‘s DataStream API and Table API to write records into iceberg table. Currently,
-we only integrate iceberg with apache flink 1.11.x .</p>
+<p>Apache Iceberg supports both <a href="https://flink.apache.org/">Apache Flink</a>‘s DataStream API and Table API to write records into an Iceberg table. Currently,
+we only integrate Iceberg with Apache Flink 1.11.x.</p>
<table>
<thead>
<tr>
@@ -547,33 +548,30 @@ we only integrate iceberg with apache flink 1.11.x .</p>
</tr>
</tbody>
</table>
-<h2 id="preparation">Preparation<a class="headerlink" href="#preparation" title="Permanent link">¶</a></h2>
+<h2 id="preparation-when-using-flink-sql-client">Preparation when using Flink SQL Client<a class="headerlink" href="#preparation-when-using-flink-sql-client" title="Permanent link">¶</a></h2>
<p>To create iceberg table in flink, we recommend to use <a href="https://ci.apache.org/projects/flink/flink-docs-stable/dev/table/sqlClient.html">Flink SQL Client</a> because it’s easier for users to understand the concepts.</p>
<p>Step.1 Downloading the flink 1.11.x binary package from the apache flink <a href="https://flink.apache.org/downloads.html">download page</a>. We now use scala 2.12 to archive the apache iceberg-flink-runtime jar, so it’s recommended to use flink 1.11 bundled with scala 2.12.</p>
-<pre><code class="bash">wget https://downloads.apache.org/flink/flink-1.11.1/flink-1.11.1-bin-scala_2.12.tgz
+<pre><code class="language-bash">wget https://downloads.apache.org/flink/flink-1.11.1/flink-1.11.1-bin-scala_2.12.tgz
tar xzvf flink-1.11.1-bin-scala_2.12.tgz
</code></pre>
-
<p>Step.2 Start a standalone flink cluster within hadoop environment.</p>
-<pre><code class="bash"># HADOOP_HOME is your hadoop root directory after unpack the binary package.
+<pre><code class="language-bash"># HADOOP_HOME is your hadoop root directory after unpack the binary package.
export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
# Start the flink standalone cluster
./bin/start-cluster.sh
</code></pre>
-
<p>Step.3 Start the flink SQL client.</p>
<p>We’ve created a separate <code>flink-runtime</code> module in iceberg project to generate a bundled jar, which could be loaded by flink SQL client directly.</p>
<p>If we want to build the <code>flink-runtime</code> bundled jar manually, please just build the <code>iceberg</code> project and it will generate the jar under <code><iceberg-root-dir>/flink-runtime/build/libs</code>. Of course, we could also download the <code>flink-runtime</code> jar from the <a href="https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-flink-runtime/">apache official repository</a>.</p>
-<pre><code class="bash"># HADOOP_HOME is your hadoop root directory after unpack the binary package.
+<pre><code class="language-bash"># HADOOP_HOME is your hadoop root directory after unpack the binary package.
export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
./bin/sql-client.sh embedded -j <flink-runtime-directory>/iceberg-flink-runtime-xxx.jar shell
</code></pre>
-
<p>By default, iceberg has included hadoop jars for hadoop catalog. If we want to use hive catalog, we will need to load the hive jars when opening the flink sql client. Fortunately, apache flink has provided a <a href="https://repo.maven.apache.org/maven2/org/apache/flink/flink-sql-connector-hive-2.3.6_2.11/1.11.0/flink-sql-connector-hive-2.3.6_2.11-1.11.0.jar">bundled hive jar</a> for sql client. So we could open the sql client
as the following:</p>
-<pre><code class="bash"># HADOOP_HOME is your hadoop root directory after unpack the binary package.
+<pre><code class="language-bash"># HADOOP_HOME is your hadoop root directory after unpack the binary package.
export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
# wget the flink-sql-connector-hive-2.3.6_2.11-1.11.0.jar from the above bundled jar URL firstly.
@@ -584,12 +582,81 @@ export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
-j <hive-bundlded-jar-directory>/flink-sql-connector-hive-2.3.6_2.11-1.11.0.jar \
shell
</code></pre>
+<h2 id="preparation-when-using-flinks-python-api">Preparation when using Flink’s Python API<a class="headerlink" href="#preparation-when-using-flinks-python-api" title="Permanent link">¶</a></h2>
+<p>Install the Apache Flink dependency using <code>pip</code></p>
+<pre><code class="language-python">pip install apache-flink==1.11.1
+</code></pre>
+<p>In order for <code>pyflink</code> to function properly, it needs to have access to all Hadoop jars. For <code>pyflink</code>
+we need to copy those Hadoop jars to the installation directory of <code>pyflink</code>, which can be found under
+<code><PYTHON_ENV_INSTALL_DIR>/site-packages/pyflink/lib/</code> (see also a mention of this on
+the <a href="http://mail-archives.apache.org/mod_mbox/flink-user/202105.mbox/%3C3D98BDD2-89B1-42F5-B6F4-6C06A038F978%40gmail.com%3E">Flink ML</a>).
+We can use the following short Python script to copy all Hadoop jars (you need to make sure that <code>HADOOP_HOME</code>
+points to your Hadoop installation):</p>
+<pre><code class="language-python">import os
+import shutil
+import site
+
+
+def copy_all_hadoop_jars_to_pyflink():
+ if not os.getenv("HADOOP_HOME"):
+ raise Exception("The HADOOP_HOME env var must be set and point to a valid Hadoop installation")
+
+ jar_files = []
+
+ def find_pyflink_lib_dir():
+ for dir in site.getsitepackages():
+ package_dir = os.path.join(dir, "pyflink", "lib")
+ if os.path.exists(package_dir):
+ return package_dir
+ return None
+ for root, _, files in os.walk(os.getenv("HADOOP_HOME")):
+ for file in files:
+ if file.endswith(".jar"):
+ jar_files.append(os.path.join(root, file))
+
+ pyflink_lib_dir = find_pyflink_lib_dir()
+
+ num_jar_files = len(jar_files)
+ print(f"Copying {num_jar_files} Hadoop jar files to pyflink's lib directory at {pyflink_lib_dir}")
+ for jar in jar_files:
+ shutil.copy(jar, pyflink_lib_dir)
+
+
+if __name__ == '__main__':
+ copy_all_hadoop_jars_to_pyflink()
+</code></pre>
+<p>Once the script finished, you should see output similar to</p>
+<pre><code>Copying 645 Hadoop jar files to pyflink's lib directory at <PYTHON_DIR>/lib/python3.8/site-packages/pyflink/lib
+</code></pre>
+<p>Now we need to provide a <code>file://</code> path to the <code>iceberg-flink-runtime</code> jar, which we can either get by building the project
+and looking at <code><iceberg-root-dir>/flink-runtime/build/libs</code>, or downloading it from the <a href="https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-flink-runtime/">Apache official repository</a>.
+Third-party libs can be added to <code>pyflink</code> via <code>env.add_jars("file:///my/jar/path/connector.jar")</code> / <code>table_env.get_config().get_configuration().set_string("pipeline.jars", "file:///my/jar/path/connector.jar")</code>, which is also mentioned in the official <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.13/docs/dev/python/dependency_management/">docs</a>.
+In our example we’re using <code>env.add_jars(..)</code> as shown below:</p>
+<pre><code class="language-python">import os
+
+from pyflink.datastream import StreamExecutionEnvironment
+
+env = StreamExecutionEnvironment.get_execution_environment()
+iceberg_flink_runtime_jar = os.path.join(os.getcwd(), "iceberg-flink-runtime-0.11.1.jar")
+
+env.add_jars("file://{}".format(iceberg_flink_runtime_jar))
+</code></pre>
+<p>Once we reached this point, we can then create a <code>StreamTableEnvironment</code> and execute Flink SQL statements.
+The below example shows how to create a custom catalog via the Python Table API:</p>
+<pre><code class="language-python">from pyflink.table import StreamTableEnvironment
+table_env = StreamTableEnvironment.create(env)
+table_env.execute_sql("CREATE CATALOG my_catalog WITH ("
+ "'type'='iceberg', "
+ "'catalog-impl'='com.my.custom.CatalogImpl', "
+ "'my-additional-catalog-config'='my-value')")
+</code></pre>
+<p>For more details, please refer to the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.13/docs/dev/python/table/intro_to_table_api/">Python Table API</a>.</p>
<h2 id="creating-catalogs-and-using-catalogs">Creating catalogs and using catalogs.<a class="headerlink" href="#creating-catalogs-and-using-catalogs" title="Permanent link">¶</a></h2>
<p>Flink 1.11 support to create catalogs by using flink sql.</p>
<h3 id="hive-catalog">Hive catalog<a class="headerlink" href="#hive-catalog" title="Permanent link">¶</a></h3>
<p>This creates an iceberg catalog named <code>hive_catalog</code> that loads tables from a hive metastore:</p>
-<pre><code class="sql">CREATE CATALOG hive_catalog WITH (
+<pre><code class="language-sql">CREATE CATALOG hive_catalog WITH (
'type'='iceberg',
'catalog-type'='hive',
'uri'='thrift://localhost:9083',
@@ -598,7 +665,6 @@ export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
'warehouse'='hdfs://nn:8020/warehouse/path'
);
</code></pre>
-
<ul>
<li><code>type</code>: Please just use <code>iceberg</code> for iceberg table format. (Required)</li>
<li><code>catalog-type</code>: Iceberg currently support <code>hive</code> or <code>hadoop</code> catalog type. (Required)</li>
@@ -607,17 +673,17 @@ export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
<li><code>property-version</code>: Version number to describe the property version. This property can be used for backwards compatibility in case the property format changes. The current property version is <code>1</code>. (Optional)</li>
<li><code>warehouse</code>: The Hive warehouse location, users should specify this path if neither set the <code>hive-conf-dir</code> to specify a location containing a <code>hive-site.xml</code> configuration file nor add a correct <code>hive-site.xml</code> to classpath.</li>
<li><code>hive-conf-dir</code>: Path to a directory containing a <code>hive-site.xml</code> configuration file which will be used to provide custom Hive configuration values. The value of <code>hive.metastore.warehouse.dir</code> from <code><hive-conf-dir>/hive-site.xml</code> (or hive configure file from classpath) will be overwrote with the <code>warehouse</code> value if setting both <code>hive-conf-dir</code> and <code>warehouse</code> when creating iceberg catalog.</li>
+<li><code>cache-enabled</code>: Whether to enable catalog cache, default value is <code>true</code></li>
</ul>
<h3 id="hadoop-catalog">Hadoop catalog<a class="headerlink" href="#hadoop-catalog" title="Permanent link">¶</a></h3>
<p>Iceberg also supports a directory-based catalog in HDFS that can be configured using <code>'catalog-type'='hadoop'</code>:</p>
-<pre><code class="sql">CREATE CATALOG hadoop_catalog WITH (
+<pre><code class="language-sql">CREATE CATALOG hadoop_catalog WITH (
'type'='iceberg',
'catalog-type'='hadoop',
'warehouse'='hdfs://nn:8020/warehouse/path',
'property-version'='1'
);
</code></pre>
-
<ul>
<li><code>warehouse</code>: The HDFS directory to store metadata files and data files. (Required)</li>
</ul>
@@ -625,96 +691,85 @@ export HADOOP_CLASSPATH=`$HADOOP_HOME/bin/hadoop classpath`
<h3 id="custom-catalog">Custom catalog<a class="headerlink" href="#custom-catalog" title="Permanent link">¶</a></h3>
<p>Flink also supports loading a custom Iceberg <code>Catalog</code> implementation by specifying the <code>catalog-impl</code> property.
When <code>catalog-impl</code> is set, the value of <code>catalog-type</code> is ignored. Here is an example:</p>
-<pre><code class="sql">CREATE CATALOG my_catalog WITH (
+<pre><code class="language-sql">CREATE CATALOG my_catalog WITH (
'type'='iceberg',
'catalog-impl'='com.my.custom.CatalogImpl',
'my-additional-catalog-config'='my-value'
);
</code></pre>
-
<h3 id="create-through-yaml-config">Create through YAML config<a class="headerlink" href="#create-through-yaml-config" title="Permanent link">¶</a></h3>
<p>Catalogs can be registered in <code>sql-client-defaults.yaml</code> before starting the SQL client. Here is an example:</p>
-<pre><code class="yaml">catalogs:
+<pre><code class="language-yaml">catalogs:
- name: my_catalog
type: iceberg
catalog-type: hadoop
warehouse: hdfs://nn:8020/warehouse/path
</code></pre>
-
<h2 id="ddl-commands">DDL commands<a class="headerlink" href="#ddl-commands" title="Permanent link">¶</a></h2>
<h3 id="create-database"><code>CREATE DATABASE</code><a class="headerlink" href="#create-database" title="Permanent link">¶</a></h3>
<p>By default, iceberg will use the <code>default</code> database in flink. Using the following example to create a separate database if we don’t want to create tables under the <code>default</code> database:</p>
-<pre><code class="sql">CREATE DATABASE iceberg_db;
+<pre><code class="language-sql">CREATE DATABASE iceberg_db;
USE iceberg_db;
</code></pre>
-
<h3 id="create-table"><code>CREATE TABLE</code><a class="headerlink" href="#create-table" title="Permanent link">¶</a></h3>
-<pre><code class="sql">CREATE TABLE hive_catalog.default.sample (
+<pre><code class="language-sql">CREATE TABLE hive_catalog.default.sample (
id BIGINT COMMENT 'unique id',
data STRING
);
</code></pre>
-
<p>Table create commands support the most commonly used <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/sql/create.html#create-table">flink create clauses</a> now, including: </p>
<ul>
<li><code>PARTITION BY (column1, column2, ...)</code> to configure partitioning, apache flink does not yet support hidden partitioning.</li>
<li><code>COMMENT 'table document'</code> to set a table description.</li>
-<li><code>WITH ('key'='value', ...)</code> to set <a href="../configuration/">table configuration</a> which will be stored in apache iceberg table properties.</li>
+<li><code>WITH ('key'='value', ...)</code> to set <a href="configuration.html">table configuration</a> which will be stored in apache iceberg table properties.</li>
</ul>
<p>Currently, it does not support computed column, primary key and watermark definition etc.</p>
<h3 id="partitioned-by"><code>PARTITIONED BY</code><a class="headerlink" href="#partitioned-by" title="Permanent link">¶</a></h3>
<p>To create a partition table, use <code>PARTITIONED BY</code>:</p>
-<pre><code class="sql">CREATE TABLE hive_catalog.default.sample (
+<pre><code class="language-sql">CREATE TABLE hive_catalog.default.sample (
id BIGINT COMMENT 'unique id',
data STRING
) PARTITIONED BY (data);
</code></pre>
-
<p>Apache Iceberg support hidden partition but apache flink don’t support partitioning by a function on columns, so we’ve no way to support hidden partition in flink DDL now, we will improve apache flink DDL in future.</p>
<h3 id="create-table-like"><code>CREATE TABLE LIKE</code><a class="headerlink" href="#create-table-like" title="Permanent link">¶</a></h3>
<p>To create a table with the same schema, partitioning, and table properties as another table, use <code>CREATE TABLE LIKE</code>.</p>
-<pre><code class="sql">CREATE TABLE hive_catalog.default.sample (
+<pre><code class="language-sql">CREATE TABLE hive_catalog.default.sample (
id BIGINT COMMENT 'unique id',
data STRING
);
CREATE TABLE hive_catalog.default.sample_like LIKE hive_catalog.default.sample;
</code></pre>
-
<p>For more details, refer to the <a href="https://ci.apache.org/projects/flink/flink-docs-release-1.11/dev/table/sql/create.html#create-table">Flink <code>CREATE TABLE</code> documentation</a>.</p>
<h3 id="alter-table"><code>ALTER TABLE</code><a class="headerlink" href="#alter-table" title="Permanent link">¶</a></h3>
<p>Iceberg only support altering table properties in flink 1.11 now.</p>
-<pre><code class="sql">ALTER TABLE hive_catalog.default.sample SET ('write.format.default'='avro')
+<pre><code class="language-sql">ALTER TABLE hive_catalog.default.sample SET ('write.format.default'='avro')
</code></pre>
-
<h3 id="alter-table-rename-to"><code>ALTER TABLE .. RENAME TO</code><a class="headerlink" href="#alter-table-rename-to" title="Permanent link">¶</a></h3>
-<pre><code class="sql">ALTER TABLE hive_catalog.default.sample RENAME TO hive_catalog.default.new_sample;
+<pre><code class="language-sql">ALTER TABLE hive_catalog.default.sample RENAME TO hive_catalog.default.new_sample;
</code></pre>
-
<h3 id="drop-table"><code>DROP TABLE</code><a class="headerlink" href="#drop-table" title="Permanent link">¶</a></h3>
<p>To delete a table, run:</p>
-<pre><code class="sql">DROP TABLE hive_catalog.default.sample;
+<pre><code class="language-sql">DROP TABLE hive_catalog.default.sample;
</code></pre>
-
<h2 id="querying-with-sql">Querying with SQL<a class="headerlink" href="#querying-with-sql" title="Permanent link">¶</a></h2>
<p>Iceberg support both streaming and batch read in flink now. we could execute the following sql command to switch the execute type from ‘streaming’ mode to ‘batch’ mode, and vice versa:</p>
-<pre><code class="sql">-- Execute the flink job in streaming mode for current session context
+<pre><code class="language-sql">-- Execute the flink job in streaming mode for current session context
SET execution.type = streaming
-- Execute the flink job in batch mode for current session context
SET execution.type = batch
</code></pre>
-
<h3 id="flink-batch-read">Flink batch read<a class="headerlink" href="#flink-batch-read" title="Permanent link">¶</a></h3>
<p>If want to check all the rows in iceberg table by submitting a flink <strong>batch</strong> job, you could execute the following sentences:</p>
-<pre><code class="sql">-- Execute the flink job in batch mode for current session context
+<pre><code class="language-sql">-- Execute the flink job in batch mode for current session context
SET execution.type = batch ;
SELECT * FROM sample ;
</code></pre>
-
<h3 id="flink-streaming-read">Flink streaming read<a class="headerlink" href="#flink-streaming-read" title="Permanent link">¶</a></h3>
<p>Iceberg supports processing incremental data in flink streaming jobs which starts from a historical snapshot-id:</p>
-<pre><code class="sql">-- Submit the flink job in streaming mode for current session.
+<pre><code class="language-sql">-- Submit the flink job in streaming mode for current session.
SET execution.type = streaming ;
-- Enable this switch because streaming read SQL will provide few job options in flink SQL hint options.
@@ -726,7 +781,6 @@ SELECT * FROM sample /*+ OPTIONS('streaming'='true', 'monitor-interval'='1s')*/
-- Read all incremental data starting from the snapshot-id '3821550127947089987' (records from this snapshot will be excluded).
SELECT * FROM sample /*+ OPTIONS('streaming'='true', 'monitor-interval'='1s', 'start-snapshot-id'='3821550127947089987')*/ ;
</code></pre>
-
<p>Those are the options that could be set in flink SQL hint options for streaming job:</p>
<ul>
<li>monitor-interval: time interval for consecutively monitoring newly committed data files (default value: ‘1s’).</li>
@@ -736,31 +790,28 @@ SELECT * FROM sample /*+ OPTIONS('streaming'='true', 'monitor-interval'='1s', 's
<p>Iceberg support both <code>INSERT INTO</code> and <code>INSERT OVERWRITE</code> in flink 1.11 now.</p>
<h3 id="insert-into"><code>INSERT INTO</code><a class="headerlink" href="#insert-into" title="Permanent link">¶</a></h3>
<p>To append new data to a table with a flink streaming job, use <code>INSERT INTO</code>:</p>
-<pre><code class="sql">INSERT INTO hive_catalog.default.sample VALUES (1, 'a');
+<pre><code class="language-sql">INSERT INTO hive_catalog.default.sample VALUES (1, 'a');
INSERT INTO hive_catalog.default.sample SELECT id, data from other_kafka_table;
</code></pre>
-
<h3 id="insert-overwrite"><code>INSERT OVERWRITE</code><a class="headerlink" href="#insert-overwrite" title="Permanent link">¶</a></h3>
<p>To replace data in the table with the result of a query, use <code>INSERT OVERWRITE</code> in batch job (flink streaming job does not support <code>INSERT OVERWRITE</code>). Overwrites are atomic operations for Iceberg tables.</p>
<p>Partitions that have rows produced by the SELECT query will be replaced, for example:</p>
-<pre><code class="sql">INSERT OVERWRITE sample VALUES (1, 'a');
+<pre><code class="language-sql">INSERT OVERWRITE sample VALUES (1, 'a');
</code></pre>
-
<p>Iceberg also support overwriting given partitions by the <code>select</code> values:</p>
-<pre><code class="sql">INSERT OVERWRITE hive_catalog.default.sample PARTITION(data='a') SELECT 6;
+<pre><code class="language-sql">INSERT OVERWRITE hive_catalog.default.sample PARTITION(data='a') SELECT 6;
</code></pre>
-
<p>For a partitioned iceberg table, when all the partition columns are set a value in <code>PARTITION</code> clause, it is inserting into a static partition, otherwise if partial partition columns (prefix part of all partition columns) are set a value in <code>PARTITION</code> clause, it is writing the query result into a dynamic partition.
For an unpartitioned iceberg table, its data will be completely overwritten by <code>INSERT OVERWRITE</code>.</p>
<h2 id="reading-with-datastream">Reading with DataStream<a class="headerlink" href="#reading-with-datastream" title="Permanent link">¶</a></h2>
<p>Iceberg support streaming or batch read in Java API now.</p>
<h3 id="batch-read">Batch Read<a class="headerlink" href="#batch-read" title="Permanent link">¶</a></h3>
<p>This example will read all records from iceberg table and then print to the stdout console in flink batch job:</p>
-<pre><code class="java">StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
+<pre><code class="language-java">StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
TableLoader tableLoader = TableLoader.fromHadooptable("hdfs://nn:8020/warehouse/path");
DataStream<RowData> batch = FlinkSource.forRowData()
.env(env)
- .tableLoader(loader)
+ .tableLoader(tableLoader)
.streaming(false)
.build();
@@ -770,16 +821,15 @@ batch.print();
// Submit and execute this batch read job.
env.execute("Test Iceberg Batch Read");
</code></pre>
-
<h3 id="streaming-read">Streaming read<a class="headerlink" href="#streaming-read" title="Permanent link">¶</a></h3>
<p>This example will read incremental records which start from snapshot-id ‘3821550127947089987’ and print to stdout console in flink streaming job:</p>
-<pre><code class="java">StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
-TableLoader tableLoader = TableLoader.fromHadooptable("hdfs://nn:8020/warehouse/path");
+<pre><code class="language-java">StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment();
+TableLoader tableLoader = TableLoader.fromHadoopTable("hdfs://nn:8020/warehouse/path");
DataStream<RowData> stream = FlinkSource.forRowData()
.env(env)
- .tableLoader(loader)
+ .tableLoader(tableLoader)
.streaming(true)
- .startSnapshotId(3821550127947089987)
+ .startSnapshotId(3821550127947089987L)
.build();
// Print all records to stdout.
@@ -788,57 +838,51 @@ stream.print();
// Submit and execute this streaming read job.
env.execute("Test Iceberg Batch Read");
</code></pre>
-
<p>There are other options that we could set by Java API, please see the <a href="./javadoc/master/org/apache/iceberg/flink/source/FlinkSource.html">FlinkSource#Builder</a>.</p>
<h2 id="writing-with-datastream">Writing with DataStream<a class="headerlink" href="#writing-with-datastream" title="Permanent link">¶</a></h2>
<p>Iceberg support writing to iceberg table from different DataStream input.</p>
<h3 id="appending-data">Appending data.<a class="headerlink" href="#appending-data" title="Permanent link">¶</a></h3>
<p>we have supported writing <code>DataStream<RowData></code> and <code>DataStream<Row></code> to the sink iceberg table natively.</p>
-<pre><code class="java">StreamExecutionEnvironment env = ...;
+<pre><code class="language-java">StreamExecutionEnvironment env = ...;
DataStream<RowData> input = ... ;
Configuration hadoopConf = new Configuration();
-TableLoader tableLoader = TableLoader.fromHadooptable("hdfs://nn:8020/warehouse/path");
+TableLoader tableLoader = TableLoader.fromHadoopTable("hdfs://nn:8020/warehouse/path", hadoopConf);
FlinkSink.forRowData(input)
.tableLoader(tableLoader)
- .hadoopConf(hadoopConf)
.build();
env.execute("Test Iceberg DataStream");
</code></pre>
-
<p>The iceberg API also allows users to write generic <code>DataStream<T></code> to iceberg table, more example could be found in this <a href="https://github.com/apache/iceberg/blob/master/flink/src/test/java/org/apache/iceberg/flink/sink/TestFlinkIcebergSink.java">unit test</a>.</p>
<h3 id="overwrite-data">Overwrite data<a class="headerlink" href="#overwrite-data" title="Permanent link">¶</a></h3>
<p>To overwrite the data in existing iceberg table dynamically, we could set the <code>overwrite</code> flag in FlinkSink builder.</p>
-<pre><code class="java">StreamExecutionEnvironment env = ...;
+<pre><code class="language-java">StreamExecutionEnvironment env = ...;
DataStream<RowData> input = ... ;
Configuration hadoopConf = new Configuration();
-TableLoader tableLoader = TableLoader.fromHadooptable("hdfs://nn:8020/warehouse/path");
+TableLoader tableLoader = TableLoader.fromHadoopTable("hdfs://nn:8020/warehouse/path", hadoopConf);
FlinkSink.forRowData(input)
.tableLoader(tableLoader)
.overwrite(true)
- .hadoopConf(hadoopConf)
.build();
env.execute("Test Iceberg DataStream");
</code></pre>
-
<h2 id="inspecting-tables">Inspecting tables.<a class="headerlink" href="#inspecting-tables" title="Permanent link">¶</a></h2>
-<p>Iceberg does not support inspecting table in flink sql now, we need to use <a href="../api/">iceberg’s Java API</a> to read iceberg’s meta data to get those table information.</p>
+<p>Iceberg does not support inspecting table in flink sql now, we need to use <a href="api.html">iceberg’s Java API</a> to read iceberg’s meta data to get those table information.</p>
<h2 id="rewrite-files-action">Rewrite files action.<a class="headerlink" href="#rewrite-files-action" title="Permanent link">¶</a></h2>
<p>Iceberg provides API to rewrite small files into large files by submitting flink batch job. The behavior of this flink action is the same as the spark’s <a href="./maintenance/#compact-data-files">rewriteDataFiles</a>.</p>
-<pre><code class="java">import org.apache.iceberg.flink.actions.Actions;
+<pre><code class="language-java">import org.apache.iceberg.flink.actions.Actions;
-TableLoader tableLoader = TableLoader.fromHadooptable("hdfs://nn:8020/warehouse/path");
+TableLoader tableLoader = TableLoader.fromHadoopTable("hdfs://nn:8020/warehouse/path");
Table table = tableLoader.loadTable();
RewriteDataFilesActionResult result = Actions.forTable(table)
.rewriteDataFiles()
.execute();
</code></pre>
-
<p>For more doc about options of the rewrite files action, please see <a href="./javadoc/master/org/apache/iceberg/flink/actions/RewriteDataFilesAction.html">RewriteDataFilesAction</a></p>
<h2 id="future-improvement">Future improvement.<a class="headerlink" href="#future-improvement" title="Permanent link">¶</a></h2>
<p>There are some features that we do not yet support in the current flink iceberg integration work:</p>
@@ -852,24 +896,34 @@ RewriteDataFilesActionResult result = Actions.forTable(table)
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/getting-started/index.html b/getting-started.html
similarity index 71%
rename from getting-started/index.html
rename to getting-started.html
index 41ecefe..ba32e42 100644
--- a/getting-started/index.html
+++ b/getting-started.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Getting Started - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li class="active">
- <a href="./">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../reliability/">
+ <a rel="prev" href="reliability.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../spark-configuration/">
+ <a rel="next" href="spark-configuration.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -433,19 +432,18 @@
<h1 id="getting-started">Getting Started<a class="headerlink" href="#getting-started" title="Permanent link">¶</a></h1>
<h2 id="using-iceberg-in-spark-3">Using Iceberg in Spark 3<a class="headerlink" href="#using-iceberg-in-spark-3" title="Permanent link">¶</a></h2>
-<p>The latest version of Iceberg is <a href="../releases/">0.11.1</a>.</p>
+<p>The latest version of Iceberg is <a href="releases.html">0.11.1</a>.</p>
<p>To use Iceberg in a Spark shell, use the <code>--packages</code> option:</p>
-<pre><code class="sh">spark-shell --packages org.apache.iceberg:iceberg-spark3-runtime:0.11.1
+<pre><code class="language-sh">spark-shell --packages org.apache.iceberg:iceberg-spark3-runtime:0.11.1
</code></pre>
-
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>If you want to include Iceberg in your Spark installation, add the <a href="https://search.maven.org/remotecontent?filepath=org/apache/iceberg/iceberg-spark3-runtime/0.11.1/iceberg-spark3-runtime-0.11.1.jar"><code>iceberg-spark3-runtime</code> Jar</a> to Spark’s <code>jars</code> folder.</p>
</div>
<h3 id="adding-catalogs">Adding catalogs<a class="headerlink" href="#adding-catalogs" title="Permanent link">¶</a></h3>
-<p>Iceberg comes with <a href="../spark-configuration/#catalogs">catalogs</a> that enable SQL commands to manage tables and load them by name. Catalogs are configured using properties under <code>spark.sql.catalog.(catalog_name)</code>.</p>
+<p>Iceberg comes with <a href="spark-configuration.html#catalogs">catalogs</a> that enable SQL commands to manage tables and load them by name. Catalogs are configured using properties under <code>spark.sql.catalog.(catalog_name)</code>.</p>
<p>This command creates a path-based catalog named <code>local</code> for tables under <code>$PWD/warehouse</code> and adds support for Iceberg tables to Spark’s built-in catalog:</p>
-<pre><code class="sh">spark-sql --packages org.apache.iceberg:iceberg-spark3-runtime:0.11.1 \
+<pre><code class="language-sh">spark-sql --packages org.apache.iceberg:iceberg-spark3-runtime:0.11.1\
--conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \
--conf spark.sql.catalog.spark_catalog=org.apache.iceberg.spark.SparkSessionCatalog \
--conf spark.sql.catalog.spark_catalog.type=hive \
@@ -453,49 +451,42 @@
--conf spark.sql.catalog.local.type=hadoop \
--conf spark.sql.catalog.local.warehouse=$PWD/warehouse
</code></pre>
-
<h3 id="creating-a-table">Creating a table<a class="headerlink" href="#creating-a-table" title="Permanent link">¶</a></h3>
-<p>To create your first Iceberg table in Spark, use the <code>spark-sql</code> shell or <code>spark.sql(...)</code> to run a <a href="../spark-ddl/#create-table"><code>CREATE TABLE</code></a> command:</p>
-<pre><code class="sql">-- local is the path-based catalog defined above
+<p>To create your first Iceberg table in Spark, use the <code>spark-sql</code> shell or <code>spark.sql(...)</code> to run a <a href="spark-ddl.html#create-table"><code>CREATE TABLE</code></a> command:</p>
+<pre><code class="language-sql">-- local is the path-based catalog defined above
CREATE TABLE local.db.table (id bigint, data string) USING iceberg
</code></pre>
-
<p>Iceberg catalogs support the full range of SQL DDL commands, including:</p>
<ul>
-<li><a href="../spark-ddl/#create-table"><code>CREATE TABLE ... PARTITIONED BY</code></a></li>
-<li><a href="../spark-ddl/#create-table-as-select"><code>CREATE TABLE ... AS SELECT</code></a></li>
-<li><a href="../spark-ddl/#alter-table"><code>ALTER TABLE</code></a></li>
-<li><a href="../spark-ddl/#drop-table"><code>DROP TABLE</code></a></li>
+<li><a href="spark-ddl.html#create-table"><code>CREATE TABLE ... PARTITIONED BY</code></a></li>
+<li><a href="spark-ddl.html#create-table-as-select"><code>CREATE TABLE ... AS SELECT</code></a></li>
+<li><a href="spark-ddl.html#alter-table"><code>ALTER TABLE</code></a></li>
+<li><a href="spark-ddl.html#drop-table"><code>DROP TABLE</code></a></li>
</ul>
<h3 id="writing">Writing<a class="headerlink" href="#writing" title="Permanent link">¶</a></h3>
-<p>Once your table is created, insert data using <a href="../spark-writes/#insert-into"><code>INSERT INTO</code></a>:</p>
-<pre><code class="sql">INSERT INTO local.db.table VALUES (1, 'a'), (2, 'b'), (3, 'c');
+<p>Once your table is created, insert data using <a href="spark-writes.html#insert-into"><code>INSERT INTO</code></a>:</p>
+<pre><code class="language-sql">INSERT INTO local.db.table VALUES (1, 'a'), (2, 'b'), (3, 'c');
INSERT INTO local.db.table SELECT id, data FROM source WHERE length(data) = 1;
</code></pre>
-
-<p>Iceberg also adds row-level SQL updates to Spark, <a href="../spark-writes/#merge-into"><code>MERGE INTO</code></a> and <a href="../spark-writes/#delete-from"><code>DELETE FROM</code></a>:</p>
-<pre><code class="sql">MERGE INTO local.db.target t USING (SELECT * FROM updates) u ON t.id = u.id
+<p>Iceberg also adds row-level SQL updates to Spark, <a href="spark-writes.html#merge-into"><code>MERGE INTO</code></a> and <a href="spark-writes.html#delete-from"><code>DELETE FROM</code></a>:</p>
+<pre><code class="language-sql">MERGE INTO local.db.target t USING (SELECT * FROM updates) u ON t.id = u.id
WHEN MATCHED THEN UPDATE SET t.count = t.count + u.count
WHEN NOT MATCHED THEN INSERT *
</code></pre>
-
-<p>Iceberg supports writing DataFrames using the new <a href="../spark-writes/#writing-with-dataframes">v2 DataFrame write API</a>:</p>
-<pre><code class="scala">spark.table("source").select("id", "data")
+<p>Iceberg supports writing DataFrames using the new <a href="spark-writes.html#writing-with-dataframes">v2 DataFrame write API</a>:</p>
+<pre><code class="language-scala">spark.table("source").select("id", "data")
.writeTo("local.db.table").append()
</code></pre>
-
<p>The old <code>write</code> API is supported, but <em>not</em> recommended.</p>
<h3 id="reading">Reading<a class="headerlink" href="#reading" title="Permanent link">¶</a></h3>
<p>To read with SQL, use the an Iceberg table name in a <code>SELECT</code> query:</p>
-<pre><code class="sql">SELECT count(1) as count, data
+<pre><code class="language-sql">SELECT count(1) as count, data
FROM local.db.table
GROUP BY data
</code></pre>
-
-<p>SQL is also the recommended way to <a href="../spark-queries/#inspecting-tables">inspect tables</a>. To view all of the snapshots in a table, use the <code>snapshots</code> metadata table:</p>
-<pre><code class="sql">SELECT * FROM local.db.table.snapshots
+<p>SQL is also the recommended way to <a href="spark-queries.html#inspecting-tables">inspect tables</a>. To view all of the snapshots in a table, use the <code>snapshots</code> metadata table:</p>
+<pre><code class="language-sql">SELECT * FROM local.db.table.snapshots
</code></pre>
-
<pre><code>+-------------------------+----------------+-----------+-----------+----------------------------------------------------+-----+
| committed_at | snapshot_id | parent_id | operation | manifest_list | ... |
+-------------------------+----------------+-----------+-----------+----------------------------------------------------+-----+
@@ -505,42 +496,50 @@ GROUP BY data
| ... | ... | ... | ... | ... | ... |
+-------------------------+----------------+-----------+-----------+----------------------------------------------------+-----+
</code></pre>
-
-<p><a href="../spark-queries/#querying-with-dataframes">DataFrame reads</a> are supported and can now reference tables by name using <code>spark.table</code>:</p>
-<pre><code class="scala">val df = spark.table("local.db.table")
+<p><a href="spark-queries.html#querying-with-dataframes">DataFrame reads</a> are supported and can now reference tables by name using <code>spark.table</code>:</p>
+<pre><code class="language-scala">val df = spark.table("local.db.table")
df.count()
</code></pre>
-
<h3 id="next-steps">Next steps<a class="headerlink" href="#next-steps" title="Permanent link">¶</a></h3>
<p>Next, you can learn more about Iceberg tables in Spark:</p>
<ul>
-<li><a href="../spark-ddl/">DDL commands</a>: <code>CREATE</code>, <code>ALTER</code>, and <code>DROP</code></li>
-<li><a href="../spark-queries/">Querying data</a>: <code>SELECT</code> queries and metadata tables</li>
-<li><a href="../spark-writes/">Writing data</a>: <code>INSERT INTO</code> and <code>MERGE INTO</code></li>
-<li><a href="../spark-procedures/">Maintaining tables</a> with stored procedures</li>
+<li><a href="spark-ddl.html">DDL commands</a>: <code>CREATE</code>, <code>ALTER</code>, and <code>DROP</code></li>
+<li><a href="spark-queries.html">Querying data</a>: <code>SELECT</code> queries and metadata tables</li>
+<li><a href="spark-writes.html">Writing data</a>: <code>INSERT INTO</code> and <code>MERGE INTO</code></li>
+<li><a href="spark-procedures.html">Maintaining tables</a> with stored procedures</li>
</ul></div>
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/hive.html b/hive.html
new file mode 100644
index 0000000..4b60b7b
--- /dev/null
+++ b/hive.html
@@ -0,0 +1,973 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+ <meta charset="utf-8">
+ <meta http-equiv="X-UA-Compatible" content="IE=edge">
+ <meta name="viewport" content="width=device-width, initial-scale=1.0">
+ <meta name="description" content="A table format for large, slow-moving tabular data">
+
+
+ <link rel="shortcut icon" href="img/favicon.ico">
+
+
+ <title>Hive - Apache Iceberg</title>
+
+
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
+ <link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
+
+ <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
+ <!--[if lt IE 9]>
+ <script src="https://cdn.jsdelivr.net/npm/html5shiv@3.7.3/dist/html5shiv.min.js"></script>
+ <script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
+ <![endif]-->
+
+
+
+
+</head>
+
+<body>
+
+ <div class="navbar navbar-default navbar-fixed-top" role="navigation">
+ <div class="container">
+
+ <!-- Collapsed navigation -->
+ <div class="navbar-header">
+ <!-- Expander button -->
+ <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
+ <span class="sr-only">Toggle navigation</span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ <span class="icon-bar"></span>
+ </button>
+
+
+ <!-- Main title -->
+
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
+ </div>
+
+ <!-- Expanded navigation -->
+ <div class="navbar-collapse collapse">
+ <!-- Main navigation -->
+ <ul class="nav navbar-nav">
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="index.html">About</a>
+</li>
+
+
+
+<li >
+ <a href="community.html">Community</a>
+</li>
+
+
+
+<li >
+ <a href="releases.html">Releases</a>
+</li>
+
+
+
+<li >
+ <a href="blogs.html">Blogs</a>
+</li>
+
+
+
+<li >
+ <a href="trademarks.html">Trademarks</a>
+</li>
+
+
+
+<li >
+ <a href="how-to-release.html">How to Release</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Tables <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="configuration.html">Configuration</a>
+</li>
+
+
+
+<li >
+ <a href="schemas.html">Schemas</a>
+</li>
+
+
+
+<li >
+ <a href="partitioning.html">Partitioning</a>
+</li>
+
+
+
+<li >
+ <a href="evolution.html">Table evolution</a>
+</li>
+
+
+
+<li >
+ <a href="maintenance.html">Maintenance</a>
+</li>
+
+
+
+<li >
+ <a href="performance.html">Performance</a>
+</li>
+
+
+
+<li >
+ <a href="reliability.html">Reliability</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Spark <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="getting-started.html">Getting Started</a>
+</li>
+
+
+
+<li >
+ <a href="spark-configuration.html">Configuration</a>
+</li>
+
+
+
+<li >
+ <a href="spark-ddl.html">DDL</a>
+</li>
+
+
+
+<li >
+ <a href="spark-queries.html">Queries</a>
+</li>
+
+
+
+<li >
+ <a href="spark-writes.html">Writes</a>
+</li>
+
+
+
+<li >
+ <a href="spark-procedures.html">Maintenance Procedures</a>
+</li>
+
+
+
+<li >
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
+</li>
+
+
+
+<li >
+ <a href="spark#time-travel">Time Travel</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+
+ <li >
+ <a href="https://trino.io/docs/current/connector/iceberg.html">Trino</a>
+ </li>
+
+
+
+ <li >
+ <a href="flink.html">Flink</a>
+ </li>
+
+
+
+ <li class="active">
+ <a href="hive.html">Hive</a>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Integrations <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="aws.html">AWS</a>
+</li>
+
+
+
+<li >
+ <a href="nessie.html">Nessie</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">API <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="/javadoc/">Javadoc</a>
+</li>
+
+
+
+<li >
+ <a href="api.html">Java API intro</a>
+</li>
+
+
+
+<li >
+ <a href="java-api-quickstart.html">Java Quickstart</a>
+</li>
+
+
+
+<li >
+ <a href="custom-catalog.html">Java Custom Catalog</a>
+</li>
+
+
+
+<li >
+ <a href="python-quickstart.html">Python Quickstart</a>
+</li>
+
+
+
+<li >
+ <a href="python-api-intro.html">Python API Intro</a>
+</li>
+
+
+
+<li >
+ <a href="python-feature-support.html">Python Feature Support</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">Format <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="terms.html">Definitions</a>
+</li>
+
+
+
+<li >
+ <a href="spec.html">Spec</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+
+ <li >
+ <a href="https://github.com/apache/iceberg">GitHub</a>
+ </li>
+
+
+
+ <li class="dropdown">
+ <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a>
+ <ul class="dropdown-menu">
+
+
+<li >
+ <a href="https://www.apache.org/licenses/">License</a>
+</li>
+
+
+
+<li >
+ <a href="https://www.apache.org/security/">Security</a>
+</li>
+
+
+
+<li >
+ <a href="https://www.apache.org/foundation/thanks.html">Sponsors</a>
+</li>
+
+
+
+<li >
+ <a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
+</li>
+
+
+
+<li >
+ <a href="https://www.apache.org/events/current-event.html">Events</a>
+</li>
+
+
+ </ul>
+ </li>
+
+
+ </ul>
+
+ <ul class="nav navbar-nav navbar-right">
+ <li >
+ <a rel="prev" href="flink.html">
+ <i class="fas fa-arrow-left"></i> Previous
+ </a>
+ </li>
+ <li >
+ <a rel="next" href="aws.html">
+ Next <i class="fas fa-arrow-right"></i>
+ </a>
+ </li>
+ </ul>
+ </div>
+ </div>
+</div>
+
+ <div class="container">
+
+
+ <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
+ <ul class="nav bs-sidenav">
+ <li class="first-level active"><a href="#hive">Hive</a></li>
+ <li class="second-level"><a href="#enabling-iceberg-support-in-hive">Enabling Iceberg support in Hive</a></li>
+
+ <li class="third-level"><a href="#loading-runtime-jar">Loading runtime jar</a></li>
+ <li class="third-level"><a href="#enabling-support">Enabling support</a></li>
+ <li class="second-level"><a href="#catalog-management">Catalog Management</a></li>
+
+ <li class="third-level"><a href="#global-hive-catalog">Global Hive catalog</a></li>
+ <li class="third-level"><a href="#custom-iceberg-catalogs">Custom Iceberg catalogs</a></li>
+ <li class="second-level"><a href="#ddl-commands">DDL Commands</a></li>
+
+ <li class="third-level"><a href="#create-external-table">CREATE EXTERNAL TABLE</a></li>
+ <li class="third-level"><a href="#create-table">CREATE TABLE</a></li>
+ <li class="third-level"><a href="#drop-table">DROP TABLE</a></li>
+ <li class="second-level"><a href="#querying-with-sql">Querying with SQL</a></li>
+
+ <li class="third-level"><a href="#configurations">Configurations</a></li>
+ <li class="third-level"><a href="#select">SELECT</a></li>
+ <li class="second-level"><a href="#writing-with-sql">Writing with SQL</a></li>
+
+ <li class="third-level"><a href="#configurations_1">Configurations</a></li>
+ <li class="third-level"><a href="#insert-into">INSERT INTO</a></li>
+ <li class="second-level"><a href="#type-compatibility">Type compatibility</a></li>
+
+ <li class="third-level"><a href="#hive-type-to-iceberg-type">Hive type to Iceberg type</a></li>
+ </ul>
+</div></div>
+ <div class="col-md-9" role="main">
+
+<!--
+ - Licensed to the Apache Software Foundation (ASF) under one or more
+ - contributor license agreements. See the NOTICE file distributed with
+ - this work for additional information regarding copyright ownership.
+ - The ASF licenses this file to You under the Apache License, Version 2.0
+ - (the "License"); you may not use this file except in compliance with
+ - the License. You may obtain a copy of the License at
+ -
+ - http://www.apache.org/licenses/LICENSE-2.0
+ -
+ - Unless required by applicable law or agreed to in writing, software
+ - distributed under the License is distributed on an "AS IS" BASIS,
+ - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ - See the License for the specific language governing permissions and
+ - limitations under the License.
+ -->
+
+<h1 id="hive">Hive<a class="headerlink" href="#hive" title="Permanent link">¶</a></h1>
+<p>Iceberg supports reading and writing Iceberg tables through <a href="https://hive.apache.org">Hive</a> by using a <a href="https://cwiki.apache.org/confluence/display/Hive/StorageHandlers">StorageHandler</a>.
+Here is the current compatibility matrix for Iceberg Hive support: </p>
+<table>
+<thead>
+<tr>
+<th>Feature</th>
+<th>Hive 2.x</th>
+<th>Hive 3.1.2</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>CREATE EXTERNAL TABLE</td>
+<td>✔️</td>
+<td>✔️</td>
+</tr>
+<tr>
+<td>CREATE TABLE</td>
+<td>✔️</td>
+<td>✔️</td>
+</tr>
+<tr>
+<td>DROP TABLE</td>
+<td>✔️</td>
+<td>✔️</td>
+</tr>
+<tr>
+<td>SELECT</td>
+<td>✔️ (MapReduce and Tez)</td>
+<td>✔️ (MapReduce and Tez)</td>
+</tr>
+<tr>
+<td>INSERT INTO</td>
+<td>✔️ (MapReduce only)️</td>
+<td>✔️ (MapReduce only)</td>
+</tr>
+</tbody>
+</table>
+<h2 id="enabling-iceberg-support-in-hive">Enabling Iceberg support in Hive<a class="headerlink" href="#enabling-iceberg-support-in-hive" title="Permanent link">¶</a></h2>
+<h3 id="loading-runtime-jar">Loading runtime jar<a class="headerlink" href="#loading-runtime-jar" title="Permanent link">¶</a></h3>
+<p>To enable Iceberg support in Hive, the <code>HiveIcebergStorageHandler</code> and supporting classes need to be made available on Hive’s classpath.
+These are provided by the <code>iceberg-hive-runtime</code> jar file.
+For example, if using the Hive shell, this can be achieved by issuing a statement like so:</p>
+<pre><code>add jar /path/to/iceberg-hive-runtime.jar;
+</code></pre>
+<p>There are many others ways to achieve this including adding the jar file to Hive’s auxiliary classpath so it is available by default.
+Please refer to Hive’s documentation for more information.</p>
+<h3 id="enabling-support">Enabling support<a class="headerlink" href="#enabling-support" title="Permanent link">¶</a></h3>
+<p>If the Iceberg storage handler is not in Hive’s classpath, then Hive cannot load or update the metadata for an Iceberg table when the storage handler is set.
+To avoid the appearance of broken tables in Hive, Iceberg will not add the storage handler to a table unless Hive support is enabled.
+The storage handler is kept in sync (added or removed) every time Hive engine support for the table is updated, i.e. turned on or off in the table properties.
+There are two ways to enable Hive support: globally in Hadoop Configuration and per-table using a table property.</p>
+<h4 id="hadoop-configuration">Hadoop configuration<a class="headerlink" href="#hadoop-configuration" title="Permanent link">¶</a></h4>
+<p>To enable Hive support globally for an application, set <code>iceberg.engine.hive.enabled=true</code> in its Hadoop configuration.
+For example, setting this in the <code>hive-site.xml</code> loaded by Spark will enable the storage handler for all tables created by Spark.</p>
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>When using Hive with Tez in <code>0.11.x</code> releases, you also have to disable vectorization (<code>hive.vectorized.execution.enabled=false</code>)</p>
+</div>
+<h4 id="table-property-configuration">Table property configuration<a class="headerlink" href="#table-property-configuration" title="Permanent link">¶</a></h4>
+<p>Alternatively, the property <code>engine.hive.enabled</code> can be set to <code>true</code> and added to the table properties when creating the Iceberg table.
+Here is an example of doing it programmatically:</p>
+<pre><code class="language-java">Catalog catalog = ...;
+Map<String, String> tableProperties = Maps.newHashMap();
+tableProperties.put(TableProperties.ENGINE_HIVE_ENABLED, "true"); // engine.hive.enabled=true
+catalog.createTable(tableId, schema, spec, tableProperties);
+</code></pre>
+<p>The table level configuration overrides the global Hadoop configuration.</p>
+<h2 id="catalog-management">Catalog Management<a class="headerlink" href="#catalog-management" title="Permanent link">¶</a></h2>
+<h3 id="global-hive-catalog">Global Hive catalog<a class="headerlink" href="#global-hive-catalog" title="Permanent link">¶</a></h3>
+<p>From the Hive engine’s perspective, there is only one global data catalog that is defined in the Hadoop configuration in the runtime environment.
+In contrast, Iceberg supports multiple different data catalog types such as Hive, Hadoop, AWS Glue, or custom catalog implementations.
+Iceberg also allows loading a table directly based on its path in the file system. Those tables do not belong to any catalog.
+Users might want to read these cross-catalog and path-based tables through the Hive engine for use cases like join.</p>
+<p>To support this, a table in the Hive metastore can represent three different ways of loading an Iceberg table,
+depending on the table’s <code>iceberg.catalog</code> property:</p>
+<ol>
+<li>The table will be loaded using a <code>HiveCatalog</code> that corresponds to the metastore configured in the Hive environment if no <code>iceberg.catalog</code> is set</li>
+<li>The table will be loaded using a custom catalog if <code>iceberg.catalog</code> is set to a catalog name (see below)</li>
+<li>The table can be loaded directly using the table’s root location if <code>iceberg.catalog</code> is set to <code>location_based_table</code></li>
+</ol>
+<p>For cases 2 and 3 above, users can create an overlay of an Iceberg table in the Hive metastore,
+so that different table types can work together in the same Hive environment.
+See <a href="#create-external-table">CREATE EXTERNAL TABLE</a> and <a href="#create-table">CREATE TABLE</a> for more details.</p>
+<h3 id="custom-iceberg-catalogs">Custom Iceberg catalogs<a class="headerlink" href="#custom-iceberg-catalogs" title="Permanent link">¶</a></h3>
+<p>To globally register different catalogs, set the following Hadoop configurations:</p>
+<table>
+<thead>
+<tr>
+<th>Config Key</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>iceberg.catalog.<catalog_name>.type</td>
+<td>type of catalog: <code>hive</code> or <code>hadoop</code></td>
+</tr>
+<tr>
+<td>iceberg.catalog.<catalog_name>.catalog-impl</td>
+<td>catalog implementation, must not be null if type is null</td>
+</tr>
+<tr>
+<td>iceberg.catalog.<catalog_name>.<key></td>
+<td>any config key and value pairs for the catalog</td>
+</tr>
+</tbody>
+</table>
+<p>Here are some examples using Hive CLI:</p>
+<p>Register a <code>HiveCatalog</code> called <code>another_hive</code>:</p>
+<pre><code>SET iceberg.catalog.another_hive.type=hive;
+SET iceberg.catalog.another_hive.uri=thrift://example.com:9083;
+SET iceberg.catalog.another_hive.clients=10;
+SET iceberg.catalog.another_hive.warehouse=hdfs://example.com:8020/warehouse;
+</code></pre>
+<p>Register a <code>HadoopCatalog</code> called <code>hadoop</code>:</p>
+<pre><code>SET iceberg.catalog.hadoop.type=hadoop;
+SET iceberg.catalog.hadoop.warehouse=hdfs://example.com:8020/warehouse;
+</code></pre>
+<p>Register an AWS <code>GlueCatalog</code> called <code>glue</code>:</p>
+<pre><code>SET iceberg.catalog.glue.catalog-impl=org.apache.iceberg.aws.GlueCatalog;
+SET iceberg.catalog.glue.warehouse=s3://my-bucket/my/key/prefix;
+SET iceberg.catalog.glue.lock-impl=org.apache.iceberg.aws.glue.DynamoLockManager;
+SET iceberg.catalog.glue.lock.table=myGlueLockTable;
+</code></pre>
+<h2 id="ddl-commands">DDL Commands<a class="headerlink" href="#ddl-commands" title="Permanent link">¶</a></h2>
+<h3 id="create-external-table">CREATE EXTERNAL TABLE<a class="headerlink" href="#create-external-table" title="Permanent link">¶</a></h3>
+<p>The <code>CREATE EXTERNAL TABLE</code> command is used to overlay a Hive table “on top of” an existing Iceberg table.
+Iceberg tables are created using either a <a href="javadoc/master/index.html?org/apache/iceberg/catalog/Catalog.html"><code>Catalog</code></a>,
+or an implementation of the <a href="javadoc/master/index.html?org/apache/iceberg/Tables.html"><code>Tables</code></a> interface,
+and Hive needs to be configured accordingly to operate on these different types of table.</p>
+<h4 id="hive-catalog-tables">Hive catalog tables<a class="headerlink" href="#hive-catalog-tables" title="Permanent link">¶</a></h4>
+<p>As described before, tables created by the <code>HiveCatalog</code> with Hive engine feature enabled are directly visible by the Hive engine, so there is no need to create an overlay.</p>
+<h4 id="custom-catalog-tables">Custom catalog tables<a class="headerlink" href="#custom-catalog-tables" title="Permanent link">¶</a></h4>
+<p>For a table in a registered catalog, specify the catalog name in the statement using table property <code>iceberg.catalog</code>.
+For example, the SQL below creates an overlay for a table in a <code>hadoop</code> type catalog named <code>hadoop_cat</code>:</p>
+<pre><code class="language-sql">SET iceberg.catalog.hadoop_cat.type=hadoop;
+SET iceberg.catalog.hadoop_cat.warehouse=hdfs://example.com:8020/hadoop_cat;
+
+CREATE EXTERNAL TABLE database_a.table_a
+STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
+TBLPROPERTIES ('iceberg.catalog'='hadoop_cat');
+</code></pre>
+<p>When <code>iceberg.catalog</code> is missing from both table properties and the global Hadoop configuration, <code>HiveCatalog</code> will be used as default.</p>
+<h4 id="path-based-hadoop-tables">Path-based Hadoop tables<a class="headerlink" href="#path-based-hadoop-tables" title="Permanent link">¶</a></h4>
+<p>Iceberg tables created using <code>HadoopTables</code> are stored entirely in a directory in a filesystem like HDFS.
+These tables are considered to have no catalog.
+To indicate that, set <code>iceberg.catalog</code> property to <code>location_based_table</code>. For example:</p>
+<pre><code class="language-sql">CREATE EXTERNAL TABLE table_a
+STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
+LOCATION 'hdfs://some_bucket/some_path/table_a'
+TBLPROPERTIES ('iceberg.catalog'='location_based_table');
+</code></pre>
+<h3 id="create-table">CREATE TABLE<a class="headerlink" href="#create-table" title="Permanent link">¶</a></h3>
+<p>Hive also supports directly creating a new Iceberg table through <code>CREATE TABLE</code> statement. For example:</p>
+<pre><code class="language-sql">CREATE TABLE database_a.table_a (
+ id bigint, name string
+) PARTITIONED BY (
+ dept string
+) STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler';
+</code></pre>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>to Hive, the table appears to be unpartitioned although the underlying Iceberg table is partitioned.</p>
+</div>
+<div class="admonition note">
+<p class="admonition-title">Note</p>
+<p>Due to the limitation of Hive <code>PARTITIONED BY</code> syntax, if you use Hive <code>CREATE TABLE</code>,
+currently you can only partition by columns, which is translated to Iceberg identity partition transform.
+You cannot partition by other Iceberg partition transforms such as <code>days(timestamp)</code>.
+To create table with all partition transforms, you need to create the table with other engines like Spark or Flink.</p>
+</div>
+<h4 id="custom-catalog-table">Custom catalog table<a class="headerlink" href="#custom-catalog-table" title="Permanent link">¶</a></h4>
+<p>You can also create a new table that is managed by a custom catalog.
+For example, the following code creates a table in a custom Hadoop catalog:</p>
+<pre><code class="language-sql">SET iceberg.catalog.hadoop_cat.type=hadoop;
+SET iceberg.catalog.hadoop_cat.warehouse=hdfs://example.com:8020/hadoop_cat;
+
+CREATE TABLE database_a.table_a (
+ id bigint, name string
+) PARTITIONED BY (
+ dept string
+) STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
+TBLPROPERTIES ('iceberg.catalog'='hadoop_cat');
+</code></pre>
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>If the table to create already exists in the custom catalog, this will create a managed overlay table.
+This means technically you can omit the <code>EXTERNAL</code> keyword when creating an overlay table.
+However, this is <strong>not recommended</strong> because creating managed overlay tables could pose a risk
+to the shared data files in case of accidental drop table commands from the Hive side,
+which would unintentionally remove all the data in the table.</p>
+</div>
+<h3 id="drop-table">DROP TABLE<a class="headerlink" href="#drop-table" title="Permanent link">¶</a></h3>
+<p>Tables can be dropped using the <code>DROP TABLE</code> command:</p>
+<pre><code class="language-sql">DROP TABLE [IF EXISTS] table_name [PURGE];
+</code></pre>
+<p>You can configure purge behavior through global Hadoop configuration or Hive metastore table properties:</p>
+<table>
+<thead>
+<tr>
+<th>Config key</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>external.table.purge</td>
+<td>true</td>
+<td>if all data and metadata should be purged in a table by default</td>
+</tr>
+</tbody>
+</table>
+<p>Each Iceberg table’s default purge behavior can also be configured through Iceberg table properties:</p>
+<table>
+<thead>
+<tr>
+<th>Property</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>gc.enabled</td>
+<td>true</td>
+<td>if all data and metadata should be purged in the table by default</td>
+</tr>
+</tbody>
+</table>
+<p>When changing <code>gc.enabled</code> on the Iceberg table via <code>UpdateProperties</code>, <code>external.table.purge</code> is also updated on HMS table accordingly.
+When setting <code>external.table.purge</code> as a table prop during Hive <code>CREATE TABLE</code>, <code>gc.enabled</code> is pushed down accordingly to the Iceberg table properties.
+This makes sure that the 2 properties are always consistent at table level between Hive and Iceberg.</p>
+<div class="admonition warning">
+<p class="admonition-title">Warning</p>
+<p>Changing <code>external.table.purge</code> via Hive <code>ALTER TABLE SET TBLPROPERTIES</code> does not update <code>gc.enabled</code> on the Iceberg table.
+This is a limitation on Hive 3.1.2 because the <code>HiveMetaHook</code> doesn’t have all the hooks for alter tables yet.</p>
+</div>
+<h2 id="querying-with-sql">Querying with SQL<a class="headerlink" href="#querying-with-sql" title="Permanent link">¶</a></h2>
+<p>Here are the features highlights for Iceberg Hive read support:</p>
+<ol>
+<li><strong>Predicate pushdown</strong>: Pushdown of the Hive SQL <code>WHERE</code> clause has been implemented so that these filters are used at the Iceberg <code>TableScan</code> level as well as by the Parquet and ORC Readers.</li>
+<li><strong>Column projection</strong>: Columns from the Hive SQL <code>SELECT</code> clause are projected down to the Iceberg readers to reduce the number of columns read.</li>
+<li><strong>Hive query engines</strong>: Both the MapReduce and Tez query execution engines are supported.</li>
+</ol>
+<h3 id="configurations">Configurations<a class="headerlink" href="#configurations" title="Permanent link">¶</a></h3>
+<p>Here are the Hadoop configurations that one can adjust for the Hive reader:</p>
+<table>
+<thead>
+<tr>
+<th>Config key</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>iceberg.mr.reuse.containers</td>
+<td>false</td>
+<td>if Avro reader should reuse containers</td>
+</tr>
+<tr>
+<td>iceberg.mr.case.sensitive</td>
+<td>true</td>
+<td>if the query is case-sensitive</td>
+</tr>
+</tbody>
+</table>
+<h3 id="select">SELECT<a class="headerlink" href="#select" title="Permanent link">¶</a></h3>
+<p>You should now be able to issue Hive SQL <code>SELECT</code> queries and see the results returned from the underlying Iceberg table, for example:</p>
+<pre><code class="language-sql">SELECT * from table_a;
+</code></pre>
+<h2 id="writing-with-sql">Writing with SQL<a class="headerlink" href="#writing-with-sql" title="Permanent link">¶</a></h2>
+<h3 id="configurations_1">Configurations<a class="headerlink" href="#configurations_1" title="Permanent link">¶</a></h3>
+<p>Here are the Hadoop configurations that one can adjust for the Hive writer:</p>
+<table>
+<thead>
+<tr>
+<th>Config key</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>iceberg.mr.commit.table.thread.pool.size</td>
+<td>10</td>
+<td>the number of threads of a shared thread pool to execute parallel commits for output tables</td>
+</tr>
+<tr>
+<td>iceberg.mr.commit.file.thread.pool.size</td>
+<td>10</td>
+<td>the number of threads of a shared thread pool to execute parallel commits for files in each output table</td>
+</tr>
+</tbody>
+</table>
+<h3 id="insert-into">INSERT INTO<a class="headerlink" href="#insert-into" title="Permanent link">¶</a></h3>
+<p>Hive supports the standard single-table <code>INSERT INTO</code> operation:</p>
+<pre><code class="language-sql">INSERT INTO table_a VALUES ('a', 1);
+INSERT INTO table_a SELECT ...;
+</code></pre>
+<p>Multi-table insert is also supported, but it will not be atomic and are committed one table at a time. Partial changes will be visible during the commit process and failures can leave partial changes committed. Changes within a single table will remain atomic.</p>
+<p>Here is an example of inserting into multiple tables at once in Hive SQL:</p>
+<pre><code class="language-sql">FROM customers
+ INSERT INTO target1 SELECT customer_id, first_name
+ INSERT INTO target2 SELECT last_name, customer_id;
+</code></pre>
+<h2 id="type-compatibility">Type compatibility<a class="headerlink" href="#type-compatibility" title="Permanent link">¶</a></h2>
+<p>Hive and Iceberg support different set of types. Iceberg can perform type conversion automatically, but not for all combinations,
+so you may want to understand the type conversion in Iceberg in prior to design the types of columns in your tables.
+You can enable auto-conversion through Hadoop configuration (not enabled by default):</p>
+<table>
+<thead>
+<tr>
+<th>Config key</th>
+<th>Default</th>
+<th>Description</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>iceberg.mr.schema.auto.conversion</td>
+<td>false</td>
+<td>if Hive should perform type auto-conversion</td>
+</tr>
+</tbody>
+</table>
+<h3 id="hive-type-to-iceberg-type">Hive type to Iceberg type<a class="headerlink" href="#hive-type-to-iceberg-type" title="Permanent link">¶</a></h3>
+<p>This type conversion table describes how Hive types are converted to the Iceberg types.
+The conversion applies on both creating Iceberg table and writing to Iceberg table via Hive.</p>
+<table>
+<thead>
+<tr>
+<th>Hive</th>
+<th>Iceberg</th>
+<th>Notes</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td>boolean</td>
+<td>boolean</td>
+<td></td>
+</tr>
+<tr>
+<td>short</td>
+<td>integer</td>
+<td>auto-conversion</td>
+</tr>
+<tr>
+<td>byte</td>
+<td>integer</td>
+<td>auto-conversion</td>
+</tr>
+<tr>
+<td>integer</td>
+<td>integer</td>
+<td></td>
+</tr>
+<tr>
+<td>long</td>
+<td>long</td>
+<td></td>
+</tr>
+<tr>
+<td>float</td>
+<td>float</td>
+<td></td>
+</tr>
+<tr>
+<td>double</td>
+<td>double</td>
+<td></td>
+</tr>
+<tr>
+<td>date</td>
+<td>date</td>
+<td></td>
+</tr>
+<tr>
+<td>timestamp</td>
+<td>timestamp without timezone</td>
+<td></td>
+</tr>
+<tr>
+<td>timestamplocaltz</td>
+<td>timestamp with timezone</td>
+<td>Hive 3 only</td>
+</tr>
+<tr>
+<td>interval_year_month</td>
+<td></td>
+<td>not supported</td>
+</tr>
+<tr>
+<td>interval_day_time</td>
+<td></td>
+<td>not supported</td>
+</tr>
+<tr>
+<td>char</td>
+<td>string</td>
+<td>auto-conversion</td>
+</tr>
+<tr>
+<td>varchar</td>
+<td>string</td>
+<td>auto-conversion</td>
+</tr>
+<tr>
+<td>string</td>
+<td>string</td>
+<td></td>
+</tr>
+<tr>
+<td>binary</td>
+<td>binary</td>
+<td></td>
+</tr>
+<tr>
+<td>decimal</td>
+<td>decimal</td>
+<td></td>
+</tr>
+<tr>
+<td>struct</td>
+<td>struct</td>
+<td></td>
+</tr>
+<tr>
+<td>list</td>
+<td>list</td>
+<td></td>
+</tr>
+<tr>
+<td>map</td>
+<td>map</td>
+<td></td>
+</tr>
+<tr>
+<td>union</td>
+<td></td>
+<td>not supported</td>
+</tr>
+</tbody>
+</table></div>
+
+
+ </div>
+
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
+
+
+
+ </footer>
+
+ <script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
+ <script>hljs.initHighlightingOnLoad();</script>
+
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
+
+ <div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
+ <div class="modal-dialog">
+ <div class="modal-content">
+ <div class="modal-header">
+ <h4 class="modal-title" id="keyboardModalLabel">Keyboard Shortcuts</h4>
+ <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button>
+ </div>
+ <div class="modal-body">
+ <table class="table">
+ <thead>
+ <tr>
+ <th style="width: 20%;">Keys</th>
+ <th>Action</th>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td class="help shortcut"><kbd>?</kbd></td>
+ <td>Open this help</td>
+ </tr>
+ <tr>
+ <td class="next shortcut"><kbd>n</kbd></td>
+ <td>Next page</td>
+ </tr>
+ <tr>
+ <td class="prev shortcut"><kbd>p</kbd></td>
+ <td>Previous page</td>
+ </tr>
+ <tr>
+ <td class="search shortcut"><kbd>s</kbd></td>
+ <td>Search</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <div class="modal-footer">
+ </div>
+ </div>
+ </div>
+</div>
+ </body>
+
+</html>
diff --git a/hive/index.html b/hive/index.html
deleted file mode 100644
index 6c1e4cc..0000000
--- a/hive/index.html
+++ /dev/null
@@ -1,581 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-
-<head>
- <meta charset="utf-8">
- <meta http-equiv="X-UA-Compatible" content="IE=edge">
- <meta name="viewport" content="width=device-width, initial-scale=1.0">
- <meta name="description" content="A table format for large, slow-moving tabular data">
-
-
- <link rel="../img/favicon.ico">
-
-
- <title>Hive - Apache Iceberg</title>
-
-
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
- <link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
-
- <!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
- <!--[if lt IE 9]>
- <script src="https://cdn.jsdelivr.net/npm/html5shiv@3.7.3/dist/html5shiv.min.js"></script>
- <script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
- <![endif]-->
-
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
-
-
-
-</head>
-
-<body>
-
- <div class="navbar navbar-default navbar-fixed-top" role="navigation">
- <div class="container">
-
- <!-- Collapsed navigation -->
- <div class="navbar-header">
- <!-- Expander button -->
- <button type="button" class="navbar-toggle" data-toggle="collapse" data-target=".navbar-collapse">
- <span class="sr-only">Toggle navigation</span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- <span class="icon-bar"></span>
- </button>
-
-
- <!-- Main title -->
-
- <a class="navbar-brand" href="..">Apache Iceberg</a>
- </div>
-
- <!-- Expanded navigation -->
- <div class="navbar-collapse collapse">
- <!-- Main navigation -->
- <ul class="nav navbar-nav">
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Project <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="..">About</a>
-</li>
-
-
-
-<li >
- <a href="../community/">Community</a>
-</li>
-
-
-
-<li >
- <a href="../releases/">Releases</a>
-</li>
-
-
-
-<li >
- <a href="../blogs/">Blogs</a>
-</li>
-
-
-
-<li >
- <a href="../trademarks/">Trademarks</a>
-</li>
-
-
-
-<li >
- <a href="../how-to-release/">How to Release</a>
-</li>
-
-
- </ul>
- </li>
-
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Tables <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="../configuration/">Configuration</a>
-</li>
-
-
-
-<li >
- <a href="../schemas/">Schemas</a>
-</li>
-
-
-
-<li >
- <a href="../partitioning/">Partitioning</a>
-</li>
-
-
-
-<li >
- <a href="../evolution/">Table evolution</a>
-</li>
-
-
-
-<li >
- <a href="../maintenance/">Maintenance</a>
-</li>
-
-
-
-<li >
- <a href="../performance/">Performance</a>
-</li>
-
-
-
-<li >
- <a href="../reliability/">Reliability</a>
-</li>
-
-
- </ul>
- </li>
-
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Spark <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="../getting-started/">Getting Started</a>
-</li>
-
-
-
-<li >
- <a href="../spark-configuration/">Configuration</a>
-</li>
-
-
-
-<li >
- <a href="../spark-ddl/">DDL</a>
-</li>
-
-
-
-<li >
- <a href="../spark-queries/">Queries</a>
-</li>
-
-
-
-<li >
- <a href="../spark-writes/">Writes</a>
-</li>
-
-
-
-<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
-</li>
-
-
-
-<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
-</li>
-
-
-
-<li >
- <a href="../spark#time-travel">Time Travel</a>
-</li>
-
-
- </ul>
- </li>
-
-
-
- <li >
- <a href="https://trino.io/docs/current/connector/iceberg.html">Trino</a>
- </li>
-
-
-
- <li >
- <a href="../flink/">Flink</a>
- </li>
-
-
-
- <li class="active">
- <a href="./">Hive</a>
- </li>
-
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Integrations <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="../aws/">AWS</a>
-</li>
-
-
-
-<li >
- <a href="../nessie/">Nessie</a>
-</li>
-
-
- </ul>
- </li>
-
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">API <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="/javadoc/">Javadoc</a>
-</li>
-
-
-
-<li >
- <a href="../api/">Java API intro</a>
-</li>
-
-
-
-<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
-</li>
-
-
-
-<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
-</li>
-
-
-
-<li >
- <a href="../python-quickstart/">Python Quickstart</a>
-</li>
-
-
-
-<li >
- <a href="../python-api-intro/">Python API Intro</a>
-</li>
-
-
-
-<li >
- <a href="../python-feature-support/">Python Feature Support</a>
-</li>
-
-
- </ul>
- </li>
-
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">Format <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="../terms/">Definitions</a>
-</li>
-
-
-
-<li >
- <a href="../spec/">Spec</a>
-</li>
-
-
- </ul>
- </li>
-
-
-
- <li >
- <a href="https://github.com/apache/iceberg">GitHub</a>
- </li>
-
-
-
- <li class="dropdown">
- <a href="#" class="dropdown-toggle" data-toggle="dropdown">ASF <b class="caret"></b></a>
- <ul class="dropdown-menu">
-
-
-<li >
- <a href="https://www.apache.org/licenses/">License</a>
-</li>
-
-
-
-<li >
- <a href="https://www.apache.org/security/">Security</a>
-</li>
-
-
-
-<li >
- <a href="https://www.apache.org/foundation/thanks.html">Sponsors</a>
-</li>
-
-
-
-<li >
- <a href="https://www.apache.org/foundation/sponsorship.html">Donate</a>
-</li>
-
-
-
-<li >
- <a href="https://www.apache.org/events/current-event.html">Events</a>
-</li>
-
-
- </ul>
- </li>
-
-
- </ul>
-
- <ul class="nav navbar-nav navbar-right">
- <li >
- <a rel="prev" href="../flink/">
- <i class="fas fa-arrow-left"></i> Previous
- </a>
- </li>
- <li >
- <a rel="next" href="../aws/">
- Next <i class="fas fa-arrow-right"></i>
- </a>
- </li>
- </ul>
- </div>
- </div>
-</div>
-
- <div class="container">
-
-
- <div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
- <ul class="nav bs-sidenav">
- <li class="first-level active"><a href="#hive">Hive</a></li>
- <li class="second-level"><a href="#hive-read-support">Hive read support</a></li>
-
- <li class="third-level"><a href="#table-creation">Table creation</a></li>
- <li class="third-level"><a href="#features">Features</a></li>
- </ul>
-</div></div>
- <div class="col-md-9" role="main">
-
-<!--
- - Licensed to the Apache Software Foundation (ASF) under one or more
- - contributor license agreements. See the NOTICE file distributed with
- - this work for additional information regarding copyright ownership.
- - The ASF licenses this file to You under the Apache License, Version 2.0
- - (the "License"); you may not use this file except in compliance with
- - the License. You may obtain a copy of the License at
- -
- - http://www.apache.org/licenses/LICENSE-2.0
- -
- - Unless required by applicable law or agreed to in writing, software
- - distributed under the License is distributed on an "AS IS" BASIS,
- - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- - See the License for the specific language governing permissions and
- - limitations under the License.
- -->
-
-<h1 id="hive">Hive<a class="headerlink" href="#hive" title="Permanent link">¶</a></h1>
-<h2 id="hive-read-support">Hive read support<a class="headerlink" href="#hive-read-support" title="Permanent link">¶</a></h2>
-<p>Iceberg supports the reading of Iceberg tables from <a href="https://hive.apache.org">Hive</a> by using a <a href="https://cwiki.apache.org/confluence/display/Hive/StorageHandlers">StorageHandler</a>. Please note that only Hive 2.x versions are currently supported.</p>
-<h3 id="table-creation">Table creation<a class="headerlink" href="#table-creation" title="Permanent link">¶</a></h3>
-<p>This section explains the various steps needed in order to overlay a Hive table “on top of” an existing Iceberg table. Iceberg tables are created using either a <a href="../javadoc/master/?org/apache/iceberg/catalog/Catalog.html"><code>Catalog</code></a> or an implementation of the <a href="../javadoc/master/?org/apache/iceberg/Tables.html"><code>Tables</code></a> interface and Hive needs to be configured accordingly to read data from these different types of table.</p>
-<h4 id="add-the-iceberg-hive-runtime-jar-file-to-the-hive-classpath">Add the Iceberg Hive Runtime jar file to the Hive classpath<a class="headerlink" href="#add-the-iceberg-hive-runtime-jar-file-to-the-hive-classpath" title="Permanent link">¶</a></h4>
-<p>Regardless of the table type, the <code>HiveIcebergStorageHandler</code> and supporting classes need to be made available on Hive’s classpath. These are provided by the <code>iceberg-hive-runtime</code> jar file. For example, if using the Hive shell, this can be achieved by issuing a statement like so:</p>
-<pre><code class="sql">add jar /path/to/iceberg-hive-runtime.jar;
-</code></pre>
-
-<p>There are many others ways to achieve this including adding the jar file to Hive’s auxiliary classpath (so it is available by default) - please refer to Hive’s documentation for more information.</p>
-<h4 id="using-hadoop-tables">Using Hadoop Tables<a class="headerlink" href="#using-hadoop-tables" title="Permanent link">¶</a></h4>
-<p>Iceberg tables created using <code>HadoopTables</code> are stored entirely in a directory in a filesystem like HDFS.</p>
-<h5 id="create-an-iceberg-table">Create an Iceberg table<a class="headerlink" href="#create-an-iceberg-table" title="Permanent link">¶</a></h5>
-<p>The first step is to create an Iceberg table using the Spark/Java/Python API and <code>HadoopTables</code>. For the purposes of this documentation we will assume that the table is called <code>table_a</code> and that the table location is <code>hdfs://some_path/table_a</code>.</p>
-<h5 id="create-a-hive-table">Create a Hive table<a class="headerlink" href="#create-a-hive-table" title="Permanent link">¶</a></h5>
-<p>Now overlay a Hive table on top of this Iceberg table by issuing Hive DDL like so:</p>
-<pre><code class="sql">CREATE EXTERNAL TABLE table_a
-STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
-LOCATION 'hdfs://some_bucket/some_path/table_a';
-</code></pre>
-
-<h4 id="query-the-iceberg-table-via-hive">Query the Iceberg table via Hive<a class="headerlink" href="#query-the-iceberg-table-via-hive" title="Permanent link">¶</a></h4>
-<p>You should now be able to issue Hive SQL <code>SELECT</code> queries using the above table and see the results returned from the underlying Iceberg table.</p>
-<pre><code class="sql">SELECT * from table_a;
-</code></pre>
-
-<h4 id="using-hive-catalog">Using Hive Catalog<a class="headerlink" href="#using-hive-catalog" title="Permanent link">¶</a></h4>
-<p>Iceberg tables created using <code>HiveCatalog</code> are automatically registered with Hive.</p>
-<h5 id="create-an-iceberg-table_1">Create an Iceberg table<a class="headerlink" href="#create-an-iceberg-table_1" title="Permanent link">¶</a></h5>
-<p>The first step is to create an Iceberg table using the Spark/Java/Python API and <code>HiveCatalog</code>. For the purposes of this documentation we will assume that the table is called <code>table_b</code> and that the table location is <code>s3://some_path/table_b</code>. In order for Iceberg to correctly set up the Hive table for querying some configuration values need to be set, the two options for this are described below - you can use either or the other depending on your use case.</p>
-<h5 id="hive-configuration">Hive Configuration<a class="headerlink" href="#hive-configuration" title="Permanent link">¶</a></h5>
-<p>The value <code>iceberg.engine.hive.enabled</code> needs to be set to <code>true</code> and added to the Hive configuration file on the classpath of the application creating or modifying (altering, inserting etc.) the table. This can be done by modifying the relevant <code>hive-site.xml</code>. Alternatively this can be done programmatically like so:</p>
-<pre><code class="java">Configuration hadoopConfiguration = spark.sparkContext().hadoopConfiguration();
-hadoopConfiguration.set(ConfigProperties.ENGINE_HIVE_ENABLED, "true"); //iceberg.engine.hive.enabled=true
-HiveCatalog catalog = new HiveCatalog(hadoopConfiguration);
-...
-catalog.createTable(tableId, schema, spec);
-</code></pre>
-
-<h5 id="table-property-configuration">Table Property Configuration<a class="headerlink" href="#table-property-configuration" title="Permanent link">¶</a></h5>
-<p>The property <code>engine.hive.enabled</code> needs to be set to <code>true</code> and added to the table properties when creating the Iceberg table. This can be done like so:</p>
-<pre><code class="java"> Map<String, String> tableProperties = new HashMap<String, String>();
- tableProperties.put(TableProperties.ENGINE_HIVE_ENABLED, "true"); //engine.hive.enabled=true
- catalog.createTable(tableId, schema, spec, tableProperties);
-</code></pre>
-
-<h4 id="query-the-iceberg-table-via-hive_1">Query the Iceberg table via Hive<a class="headerlink" href="#query-the-iceberg-table-via-hive_1" title="Permanent link">¶</a></h4>
-<p>In order to query a Hive table created by either of the HiveCatalog methods described above you need to first set a Hive configuration value like so:</p>
-<pre><code class="sql">SET iceberg.mr.catalog=hive;
-</code></pre>
-
-<p>You should now be able to issue Hive SQL <code>SELECT</code> queries using the above table and see the results returned from the underlying Iceberg table.</p>
-<pre><code class="sql">SELECT * from table_b;
-</code></pre>
-
-<h4 id="using-hadoop-catalog">Using Hadoop Catalog<a class="headerlink" href="#using-hadoop-catalog" title="Permanent link">¶</a></h4>
-<p>Iceberg tables created using <code>HadoopCatalog</code> are stored entirely in a directory in a filesystem like HDFS.</p>
-<h5 id="create-an-iceberg-table_2">Create an Iceberg table<a class="headerlink" href="#create-an-iceberg-table_2" title="Permanent link">¶</a></h5>
-<p>The first step is to create an Iceberg table using the Spark/Java/Python API and <code>HadoopCatalog</code>. For the purposes of this documentation we will assume that the fully qualified table identifier is <code>database_a.table_c</code> and that the Hadoop Catalog warehouse location is <code>hdfs://some_bucket/path_to_hadoop_warehouse</code>. Iceberg will therefore create the table at the location <code>hdfs://some_bucket/path_to_hadoop_warehouse/database_a/table_c</code>.</p>
-<h5 id="create-a-hive-table_1">Create a Hive table<a class="headerlink" href="#create-a-hive-table_1" title="Permanent link">¶</a></h5>
-<p>Now overlay a Hive table on top of this Iceberg table by issuing Hive DDL like so:</p>
-<pre><code class="sql">CREATE EXTERNAL TABLE database_a.table_c
-STORED BY 'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler'
-LOCATION 'hdfs://some_bucket/path_to_hadoop_warehouse/database_a/table_c'
-TBLPROPERTIES (
- 'iceberg.mr.catalog'='hadoop',
- 'iceberg.mr.catalog.hadoop.warehouse.location'='hdfs://some_bucket/path_to_hadoop_warehouse')
-;
-</code></pre>
-
-<p>Note that the Hive database and table name <em>must</em> match the values used in the Iceberg <code>TableIdentifier</code> when the table was created. </p>
-<p>It is possible to omit either or both of the table properties but instead you will then need to set these when reading from the table. Generally it is recommended to set them at table creation time so you can query tables created by different catalogs. </p>
-<h4 id="query-the-iceberg-table-via-hive_2">Query the Iceberg table via Hive<a class="headerlink" href="#query-the-iceberg-table-via-hive_2" title="Permanent link">¶</a></h4>
-<p>You should now be able to issue Hive SQL <code>SELECT</code> queries using the above table and see the results returned from the underlying Iceberg table.</p>
-<pre><code class="sql">SELECT * from database_a.table_c;
-</code></pre>
-
-<h3 id="features">Features<a class="headerlink" href="#features" title="Permanent link">¶</a></h3>
-<h4 id="predicate-pushdown">Predicate pushdown<a class="headerlink" href="#predicate-pushdown" title="Permanent link">¶</a></h4>
-<p>Pushdown of the Hive SQL <code>WHERE</code> clause has been implemented so that these filters are used at the Iceberg TableScan level as well as by the Parquet and ORC Readers.</p>
-<h4 id="column-projection">Column Projection<a class="headerlink" href="#column-projection" title="Permanent link">¶</a></h4>
-<p>Columns from the Hive SQL <code>SELECT</code> clause are projected down to the Iceberg readers to reduce the number of columns read.</p>
-<h4 id="hive-query-engines">Hive Query Engines<a class="headerlink" href="#hive-query-engines" title="Permanent link">¶</a></h4>
-<p>Both the Map Reduce and Tez query execution engines are supported.</p></div>
-
-
- </div>
-
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
-
-
-
- </footer>
- <script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
- <script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
-
- <script src="../js/base.js"></script>
-
- <div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
- <div class="modal-dialog">
- <div class="modal-content">
- <div class="modal-header">
- <h4 class="modal-title" id="keyboardModalLabel">Keyboard Shortcuts</h4>
- <button type="button" class="close" data-dismiss="modal"><span aria-hidden="true">×</span><span class="sr-only">Close</span></button>
- </div>
- <div class="modal-body">
- <table class="table">
- <thead>
- <tr>
- <th style="width: 20%;">Keys</th>
- <th>Action</th>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td class="help shortcut"><kbd>?</kbd></td>
- <td>Open this help</td>
- </tr>
- <tr>
- <td class="next shortcut"><kbd>n</kbd></td>
- <td>Next page</td>
- </tr>
- <tr>
- <td class="prev shortcut"><kbd>p</kbd></td>
- <td>Previous page</td>
- </tr>
- <tr>
- <td class="search shortcut"><kbd>s</kbd></td>
- <td>Search</td>
- </tr>
- </tbody>
- </table>
- </div>
- <div class="modal-footer">
- </div>
- </div>
- </div>
-</div>
- </body>
-
-</html>
diff --git a/how-to-release/index.html b/how-to-release.html
similarity index 81%
rename from how-to-release/index.html
rename to how-to-release.html
index 69f2c46..ffd0e25 100644
--- a/how-to-release/index.html
+++ b/how-to-release.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>How to Release - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li class="active">
- <a href="./">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="../java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../trademarks/">
+ <a rel="prev" href="trademarks.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../configuration/">
+ <a rel="next" href="configuration.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -448,21 +447,18 @@
<pre><code>mavenUser=yourApacheID
mavenPassword=SomePassword
</code></pre>
-
<h3 id="pgp-signing">PGP signing<a class="headerlink" href="#pgp-signing" title="Permanent link">¶</a></h3>
<p>The release scripts use the command-line <code>gpg</code> utility so that signing can use the gpg-agent and does not require writing your private key’s passphrase to a configuration file.</p>
<p>To configure gradle to sign convenience binary artifacts, add the following settings to <code>~/.gradle/gradle.properties</code>:</p>
<pre><code>signing.gnupg.keyName=Your Name (CODE SIGNING KEY)
</code></pre>
-
<p>To use <code>gpg</code> instead of <code>gpg2</code>, also set <code>signing.gnupg.executable=gpg</code></p>
<p>For more information, see the Gradle <a href="https://docs.gradle.org/current/userguide/signing_plugin.html#sec:signatory_credentials">signing documentation</a>.</p>
<h2 id="creating-a-release-candidate">Creating a release candidate<a class="headerlink" href="#creating-a-release-candidate" title="Permanent link">¶</a></h2>
<h3 id="build-the-source-release">Build the source release<a class="headerlink" href="#build-the-source-release" title="Permanent link">¶</a></h3>
<p>To create the source release artifacts, run the <code>source-release.sh</code> script with the release version and release candidate number:</p>
-<pre><code class="bash">dev/source-release.sh 0.8.1 0
+<pre><code class="language-bash">dev/source-release.sh 0.8.1 0
</code></pre>
-
<pre><code>Preparing source for apache-iceberg-0.8.1-rc0
...
Success! The release candidate is available here:
@@ -470,21 +466,18 @@ Success! The release candidate is available here:
Commit SHA1: 4b4716c76559b3cdf3487e6b60ab52950241989b
</code></pre>
-
<p>The source release script will create a candidate tag based on the HEAD revision in git and will prepare the release tarball, signature, and checksum files. It will also upload the source artifacts to SVN.</p>
<p>Note the commit SHA1 and candidate location because those will be added to the vote thread.</p>
<p>Once the source release is ready, use it to stage convenience binary artifacts in Nexus.</p>
<h3 id="build-and-stage-convenience-binaries">Build and stage convenience binaries<a class="headerlink" href="#build-and-stage-convenience-binaries" title="Permanent link">¶</a></h3>
<p>Convenience binaries are created using the source release tarball from in the last step.</p>
<p>Untar the source release and go into the release directory:</p>
-<pre><code class="bash">tar xzf apache-iceberg-0.8.1.tar.gz
+<pre><code class="language-bash">tar xzf apache-iceberg-0.8.1.tar.gz
cd apache-iceberg-0.8.1
</code></pre>
-
<p>To build and publish the convenience binaries, run the <code>dev/stage-binaries.sh</code> script. This will push to a release staging repository.</p>
<pre><code>dev/stage-binaries.sh
</code></pre>
-
<p>Next, you need to close the staging repository:</p>
<ol>
<li>Go to <a href="https://repository.apache.org/">Nexus</a> and log in</li>
@@ -497,10 +490,9 @@ cd apache-iceberg-0.8.1
</ol>
<h3 id="start-a-vote-thread">Start a VOTE thread<a class="headerlink" href="#start-a-vote-thread" title="Permanent link">¶</a></h3>
<p>The last step for a candidate is to create a VOTE thread on the dev mailing list.</p>
-<pre><code class="text">Subject: [VOTE] Release Apache Iceberg <VERSION> RC<NUM>
+<pre><code class="language-text">Subject: [VOTE] Release Apache Iceberg <VERSION> RC<NUM>
</code></pre>
-
-<pre><code class="text">Hi everyone,
+<pre><code class="language-text">Hi everyone,
I propose the following RC to be released as official Apache Iceberg <VERSION> release.
@@ -528,12 +520,10 @@ Please vote in the next 72 hours.
[ ] +0
[ ] -1 Do not release this because...
</code></pre>
-
<p>When a candidate is passed or rejected, reply with the voting result:</p>
-<pre><code class="text">Subject: [RESULT][VOTE] Release Apache Iceberg <VERSION> RC<NUM>
+<pre><code class="language-text">Subject: [RESULT][VOTE] Release Apache Iceberg <VERSION> RC<NUM>
</code></pre>
-
-<pre><code class="text">Thanks everyone who participated in the vote for Release Apache Iceberg <VERSION> RC<NUM>.
+<pre><code class="language-text">Thanks everyone who participated in the vote for Release Apache Iceberg <VERSION> RC<NUM>.
The vote result is:
@@ -543,11 +533,10 @@ The vote result is:
Therefore, the release candidate is passed/rejected.
</code></pre>
-
<h3 id="finishing-the-release">Finishing the release<a class="headerlink" href="#finishing-the-release" title="Permanent link">¶</a></h3>
<p>After the release vote has passed, you need to release the last candidate’s artifacts.</p>
<p>First, copy the source release directory to releases:</p>
-<pre><code class="bash">mkdir iceberg
+<pre><code class="language-bash">mkdir iceberg
cd iceberg
svn co https://dist.apache.org/repos/dist/dev/iceberg candidates
svn co https://dist.apache.org/repos/dist/release/iceberg releases
@@ -556,17 +545,14 @@ cd releases
svn add apache-iceberg-<VERSION>
svn ci -m 'Iceberg: Add release <VERSION>'
</code></pre>
-
<p>Next, add a release tag to the git repository based on the passing candidate tag:</p>
-<pre><code class="bash">git tag -am 'Release Apache Iceberg <VERSION>' apache-iceberg-<VERSION> apache-iceberg-<VERSION>-rcN
+<pre><code class="language-bash">git tag -am 'Release Apache Iceberg <VERSION>' apache-iceberg-<VERSION> apache-iceberg-<VERSION>-rcN
</code></pre>
-
<p>Then release the candidate repository in <a href="https://repository.apache.org/#stagingRepositories">Nexus</a>.</p>
<p>To announce the release, wait until Maven central has mirrored the Apache binaries, then update the Iceberg site and send an announcement email:</p>
-<pre><code class="text">[ANNOUNCE] Apache Iceberg release <VERSION>
+<pre><code class="language-text">[ANNOUNCE] Apache Iceberg release <VERSION>
</code></pre>
-
-<pre><code class="text">I'm please to announce the release of Apache Iceberg <VERSION>!
+<pre><code class="language-text">I'm please to announce the release of Apache Iceberg <VERSION>!
Apache Iceberg is an open table format for huge analytic datasets. Iceberg
delivers high query performance for tables with tens of petabytes of data,
@@ -583,24 +569,34 @@ Thanks to everyone for contributing!
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/img/favicon.ico b/img/favicon.ico
index e85006a..2a1daea 100644
Binary files a/img/favicon.ico and b/img/favicon.ico differ
diff --git a/img/iceberg-metadata.png b/img/iceberg-metadata.png
index b3146d3..48a1b0c 100644
Binary files a/img/iceberg-metadata.png and b/img/iceberg-metadata.png differ
diff --git a/index.html b/index.html
index 3c60cba..07ded57 100644
--- a/index.html
+++ b/index.html
@@ -8,20 +8,26 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>About - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
<link href="css/bootstrap-custom.min.css" rel="stylesheet">
<link href="css/base.min.css" rel="stylesheet">
<link href="css/cinder.min.css" rel="stylesheet">
- <link href="css/highlight.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
<link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href=".">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li class="active">
- <a href=".">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,43 +172,43 @@
<li >
- <a href="getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li >
- <a href="java-api-quickstart/">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -388,7 +387,7 @@
</a>
</li>
<li >
- <a rel="next" href="community/">
+ <a rel="next" href="community.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -402,9 +401,13 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
<ul class="nav bs-sidenav">
- <li class="first-level active"><a href="#user-experience">User experience</a></li>
- <li class="first-level "><a href="#reliability-and-performance">Reliability and performance</a></li>
- <li class="first-level "><a href="#open-standard">Open standard</a></li>
+ <li class="first-level active"><a href="#_1"></a></li>
+ <li class="second-level"><a href="#user-experience">User experience</a></li>
+
+ <li class="second-level"><a href="#reliability-and-performance">Reliability and performance</a></li>
+
+ <li class="second-level"><a href="#open-standard">Open standard</a></li>
+
</ul>
</div></div>
<div class="col-md-9" role="main">
@@ -458,21 +461,31 @@
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script src="js/bootstrap-3.0.3.min.js"></script>
- <script src="js/highlight.pack.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
+
+
<script>var base_url = "."</script>
<script src="js/base.js"></script>
diff --git a/java-api-quickstart/index.html b/java-api-quickstart.html
similarity index 80%
rename from java-api-quickstart/index.html
rename to java-api-quickstart.html
index a013b5a..ad3b3df 100644
--- a/java-api-quickstart/index.html
+++ b/java-api-quickstart.html
@@ -8,21 +8,27 @@
<meta name="description" content="A table format for large, slow-moving tabular data">
- <link rel="../img/favicon.ico">
+ <link rel="shortcut icon" href="img/favicon.ico">
<title>Java Quickstart - Apache Iceberg</title>
- <link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/all.css">
+ <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.12.0/css/v4-shims.css">
<link rel="stylesheet" href="//cdn.jsdelivr.net/npm/hack-font@3.3.0/build/web/hack.min.css">
- <link href='//fonts.googleapis.com/css?family=PT+Sans:400,400italic,700,700italic&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
+ <link href='//rsms.me/inter/inter.css' rel='stylesheet' type='text/css'>
<link href='//fonts.googleapis.com/css?family=Open+Sans:300italic,400italic,700italic,400,300,600,700&subset=latin-ext,latin' rel='stylesheet' type='text/css'>
- <link href="../css/bootstrap-custom.min.css" rel="stylesheet">
- <link href="../css/base.min.css" rel="stylesheet">
- <link href="../css/cinder.min.css" rel="stylesheet">
- <link href="../css/highlight.min.css" rel="stylesheet">
- <link href="../css/extra.css" rel="stylesheet">
+ <link href="css/bootstrap-custom.min.css" rel="stylesheet">
+ <link href="css/base.min.css" rel="stylesheet">
+ <link href="css/cinder.min.css" rel="stylesheet">
+
+
+
+ <link rel="stylesheet" href="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/styles/github.min.css">
+
+
+ <link href="css/extra.css" rel="stylesheet">
<!-- HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries -->
<!--[if lt IE 9]>
@@ -30,15 +36,6 @@
<script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->
- <script src="//ajax.googleapis.com/ajax/libs/webfont/1.6.26/webfont.js"></script>
- <script>
- WebFont.load({
- google: {
- families: ['Open Sans', 'PT Sans']
- }
- });
- </script>
-
@@ -62,7 +59,9 @@
<!-- Main title -->
- <a class="navbar-brand" href="..">Apache Iceberg</a>
+
+ <a class="navbar-brand" href=".">Apache Iceberg</a>
+
</div>
<!-- Expanded navigation -->
@@ -77,37 +76,37 @@
<li >
- <a href="..">About</a>
+ <a href="index.html">About</a>
</li>
<li >
- <a href="../community/">Community</a>
+ <a href="community.html">Community</a>
</li>
<li >
- <a href="../releases/">Releases</a>
+ <a href="releases.html">Releases</a>
</li>
<li >
- <a href="../blogs/">Blogs</a>
+ <a href="blogs.html">Blogs</a>
</li>
<li >
- <a href="../trademarks/">Trademarks</a>
+ <a href="trademarks.html">Trademarks</a>
</li>
<li >
- <a href="../how-to-release/">How to Release</a>
+ <a href="how-to-release.html">How to Release</a>
</li>
@@ -122,43 +121,43 @@
<li >
- <a href="../configuration/">Configuration</a>
+ <a href="configuration.html">Configuration</a>
</li>
<li >
- <a href="../schemas/">Schemas</a>
+ <a href="schemas.html">Schemas</a>
</li>
<li >
- <a href="../partitioning/">Partitioning</a>
+ <a href="partitioning.html">Partitioning</a>
</li>
<li >
- <a href="../evolution/">Table evolution</a>
+ <a href="evolution.html">Table evolution</a>
</li>
<li >
- <a href="../maintenance/">Maintenance</a>
+ <a href="maintenance.html">Maintenance</a>
</li>
<li >
- <a href="../performance/">Performance</a>
+ <a href="performance.html">Performance</a>
</li>
<li >
- <a href="../reliability/">Reliability</a>
+ <a href="reliability.html">Reliability</a>
</li>
@@ -173,49 +172,49 @@
<li >
- <a href="../getting-started/">Getting Started</a>
+ <a href="getting-started.html">Getting Started</a>
</li>
<li >
- <a href="../spark-configuration/">Configuration</a>
+ <a href="spark-configuration.html">Configuration</a>
</li>
<li >
- <a href="../spark-ddl/">DDL</a>
+ <a href="spark-ddl.html">DDL</a>
</li>
<li >
- <a href="../spark-queries/">Queries</a>
+ <a href="spark-queries.html">Queries</a>
</li>
<li >
- <a href="../spark-writes/">Writes</a>
+ <a href="spark-writes.html">Writes</a>
</li>
<li >
- <a href="../spark-procedures/">Maintenance Procedures</a>
+ <a href="spark-procedures.html">Maintenance Procedures</a>
</li>
<li >
- <a href="../spark-structured-streaming/">Structured Streaming</a>
+ <a href="spark-structured-streaming.html">Structured Streaming</a>
</li>
<li >
- <a href="../spark#time-travel">Time Travel</a>
+ <a href="spark#time-travel">Time Travel</a>
</li>
@@ -231,13 +230,13 @@
<li >
- <a href="../flink/">Flink</a>
+ <a href="flink.html">Flink</a>
</li>
<li >
- <a href="../hive/">Hive</a>
+ <a href="hive.html">Hive</a>
</li>
@@ -248,13 +247,13 @@
<li >
- <a href="../aws/">AWS</a>
+ <a href="aws.html">AWS</a>
</li>
<li >
- <a href="../nessie/">Nessie</a>
+ <a href="nessie.html">Nessie</a>
</li>
@@ -275,37 +274,37 @@
<li >
- <a href="../api/">Java API intro</a>
+ <a href="api.html">Java API intro</a>
</li>
<li class="active">
- <a href="./">Java Quickstart</a>
+ <a href="java-api-quickstart.html">Java Quickstart</a>
</li>
<li >
- <a href="../custom-catalog/">Java Custom Catalog</a>
+ <a href="custom-catalog.html">Java Custom Catalog</a>
</li>
<li >
- <a href="../python-quickstart/">Python Quickstart</a>
+ <a href="python-quickstart.html">Python Quickstart</a>
</li>
<li >
- <a href="../python-api-intro/">Python API Intro</a>
+ <a href="python-api-intro.html">Python API Intro</a>
</li>
<li >
- <a href="../python-feature-support/">Python Feature Support</a>
+ <a href="python-feature-support.html">Python Feature Support</a>
</li>
@@ -320,13 +319,13 @@
<li >
- <a href="../terms/">Definitions</a>
+ <a href="terms.html">Definitions</a>
</li>
<li >
- <a href="../spec/">Spec</a>
+ <a href="spec.html">Spec</a>
</li>
@@ -383,12 +382,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../api/">
+ <a rel="prev" href="api.html">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../custom-catalog/">
+ <a rel="next" href="custom-catalog.html">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -440,17 +439,16 @@
<h1 id="java-api-quickstart">Java API Quickstart<a class="headerlink" href="#java-api-quickstart" title="Permanent link">¶</a></h1>
<h2 id="create-a-table">Create a table<a class="headerlink" href="#create-a-table" title="Permanent link">¶</a></h2>
-<p>Tables are created using either a <a href="../javadoc/master/?org/apache/iceberg/catalog/Catalog.html"><code>Catalog</code></a> or an implementation of the <a href="../javadoc/master/?org/apache/iceberg/Tables.html"><code>Tables</code></a> interface.</p>
+<p>Tables are created using either a <a href="javadoc/master/index.html?org/apache/iceberg/catalog/Catalog.html"><code>Catalog</code></a> or an implementation of the <a href="javadoc/master/index.html?org/apache/iceberg/Tables.html"><code>Tables</code></a> interface.</p>
<h3 id="using-a-hive-catalog">Using a Hive catalog<a class="headerlink" href="#using-a-hive-catalog" title="Permanent link">¶</a></h3>
<p>The Hive catalog connects to a Hive MetaStore to keep track of Iceberg tables. This example uses Spark’s Hadoop configuration to get a Hive catalog:</p>
-<pre><code class="java">import org.apache.iceberg.hive.HiveCatalog;
+<pre><code class="language-java">import org.apache.iceberg.hive.HiveCatalog;
Catalog catalog = new HiveCatalog(spark.sparkContext().hadoopConfiguration());
</code></pre>
-
<p>The <code>Catalog</code> interface defines methods for working with tables, like <code>createTable</code>, <code>loadTable</code>, <code>renameTable</code>, and <code>dropTable</code>.</p>
<p>To create a table, pass an <code>Identifier</code> and a <code>Schema</code> along with other initial metadata:</p>
-<pre><code class="java">import org.apache.iceberg.Table;
+<pre><code class="language-java">import org.apache.iceberg.Table;
import org.apache.iceberg.catalog.TableIdentifier;
TableIdentifier name = TableIdentifier.of("logging", "logs");
@@ -459,21 +457,19 @@ Table table = catalog.createTable(name, schema, spec);
// or to load an existing table, use the following line
// Table table = catalog.loadTable(name);
</code></pre>
-
<p>The logs <a href="#create-a-schema">schema</a> and <a href="#create-a-partition-spec">partition spec</a> are created below.</p>
<h3 id="using-a-hadoop-catalog">Using a Hadoop catalog<a class="headerlink" href="#using-a-hadoop-catalog" title="Permanent link">¶</a></h3>
<p>A Hadoop catalog doesn’t need to connect to a Hive MetaStore, but can only be used with HDFS or similar file systems that support atomic rename. Concurrent writes with a Hadoop catalog are not safe with a local FS or S3. To create a Hadoop catalog:</p>
-<pre><code class="java">import org.apache.hadoop.conf.Configuration;
+<pre><code class="language-java">import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.hadoop.HadoopCatalog;
Configuration conf = new Configuration();
String warehousePath = "hdfs://host:8020/warehouse_path";
HadoopCatalog catalog = new HadoopCatalog(conf, warehousePath);
</code></pre>
-
<p>Like the Hive catalog, <code>HadoopCatalog</code> implements <code>Catalog</code>, so it also has methods for working with tables, like <code>createTable</code>, <code>loadTable</code>, and <code>dropTable</code>.</p>
<p>This example creates a table with Hadoop catalog:</p>
-<pre><code class="java">import org.apache.iceberg.Table;
+<pre><code class="language-java">import org.apache.iceberg.Table;
import org.apache.iceberg.catalog.TableIdentifier;
TableIdentifier name = TableIdentifier.of("logging", "logs");
@@ -482,12 +478,11 @@ Table table = catalog.createTable(name, schema, spec);
// or to load an existing table, use the following line
// Table table = catalog.loadTable(name);
</code></pre>
-
<p>The logs <a href="#create-a-schema">schema</a> and <a href="#create-a-partition-spec">partition spec</a> are created below.</p>
<h3 id="using-hadoop-tables">Using Hadoop tables<a class="headerlink" href="#using-hadoop-tables" title="Permanent link">¶</a></h3>
<p>Iceberg also supports tables that are stored in a directory in HDFS. Concurrent writes with a Hadoop tables are not safe when stored in the local FS or S3. Directory tables don’t support all catalog operations, like rename, so they use the <code>Tables</code> interface instead of <code>Catalog</code>.</p>
<p>To create a table in HDFS, use <code>HadoopTables</code>:</p>
-<pre><code class="java">import org.apache.hadoop.conf.Configuration;
+<pre><code class="language-java">import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.hadoop.HadoopTables;
import org.apache.iceberg.Table;
@@ -498,7 +493,6 @@ Table table = tables.create(schema, spec, table_location);
// or to load an existing table, use the following line
// Table table = tables.load(table_location);
</code></pre>
-
<div class="admonition warning">
<p class="admonition-title">Warning</p>
<p>Hadoop tables shouldn’t be used with file systems that do not support atomic rename. Iceberg relies on rename to synchronize concurrent commits for directory tables.</p>
@@ -507,14 +501,14 @@ Table table = tables.create(schema, spec, table_location);
<p>Spark uses both <code>HiveCatalog</code> and <code>HadoopTables</code> to load tables. Hive is used when the identifier passed to <code>load</code> or <code>save</code> is not a path, otherwise Spark assumes it is a path-based table.</p>
<p>To read and write to tables from Spark see:</p>
<ul>
-<li><a href="../spark/#reading-an-iceberg-table">Reading a table in Spark</a></li>
-<li><a href="../spark/#appending-data">Appending to a table in Spark</a></li>
-<li><a href="../spark/#overwriting-data">Overwriting data in a table in Spark</a></li>
+<li><a href="./spark.md#reading-an-iceberg-table">Reading a table in Spark</a></li>
+<li><a href="./spark.md#appending-data">Appending to a table in Spark</a></li>
+<li><a href="./spark.md#overwriting-data">Overwriting data in a table in Spark</a></li>
</ul>
<h2 id="schemas">Schemas<a class="headerlink" href="#schemas" title="Permanent link">¶</a></h2>
<h3 id="create-a-schema">Create a schema<a class="headerlink" href="#create-a-schema" title="Permanent link">¶</a></h3>
<p>This example creates a schema for a <code>logs</code> table:</p>
-<pre><code class="java">import org.apache.iceberg.Schema;
+<pre><code class="language-java">import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types;
Schema schema = new Schema(
@@ -524,61 +518,67 @@ Schema schema = new Schema(
Types.NestedField.optional(4, "call_stack", Types.ListType.ofRequired(5, Types.StringType.get()))
);
</code></pre>
-
<p>When using the Iceberg API directly, type IDs are required. Conversions from other schema formats, like Spark, Avro, and Parquet will automatically assign new IDs.</p>
<p>When a table is created, all IDs in the schema are re-assigned to ensure uniqueness.</p>
<h3 id="convert-a-schema-from-avro">Convert a schema from Avro<a class="headerlink" href="#convert-a-schema-from-avro" title="Permanent link">¶</a></h3>
<p>To create an Iceberg schema from an existing Avro schema, use converters in <code>AvroSchemaUtil</code>:</p>
-<pre><code class="java">import org.apache.avro.Schema;
+<pre><code class="language-java">import org.apache.avro.Schema;
import org.apache.avro.Schema.Parser;
import org.apache.iceberg.avro.AvroSchemaUtil;
Schema avroSchema = new Parser().parse("{\"type\": \"record\" , ... }");
Schema icebergSchema = AvroSchemaUtil.toIceberg(avroSchema);
</code></pre>
-
<h3 id="convert-a-schema-from-spark">Convert a schema from Spark<a class="headerlink" href="#convert-a-schema-from-spark" title="Permanent link">¶</a></h3>
<p>To create an Iceberg schema from an existing table, use converters in <code>SparkSchemaUtil</code>:</p>
-<pre><code class="java">import org.apache.iceberg.spark.SparkSchemaUtil;
+<pre><code class="language-java">import org.apache.iceberg.spark.SparkSchemaUtil;
Schema schema = SparkSchemaUtil.schemaForTable(sparkSession, table_name);
</code></pre>
-
<h2 id="partitioning">Partitioning<a class="headerlink" href="#partitioning" title="Permanent link">¶</a></h2>
<h3 id="create-a-partition-spec">Create a partition spec<a class="headerlink" href="#create-a-partition-spec" title="Permanent link">¶</a></h3>
<p>Partition specs describe how Iceberg should group records into data files. Partition specs are created for a table’s schema using a builder.</p>
<p>This example creates a partition spec for the <code>logs</code> table that partitions records by the hour of the log event’s timestamp and by log level:</p>
-<pre><code class="java">import org.apache.iceberg.PartitionSpec;
+<pre><code class="language-java">import org.apache.iceberg.PartitionSpec;
PartitionSpec spec = PartitionSpec.builderFor(schema)
.hour("event_time")
.identity("level")
.build();
</code></pre>
-
-<p>For more information on the different partition transforms that Iceberg offers, visit <a href="../spec/#partitioning">this page</a>.</p></div>
+<p>For more information on the different partition transforms that Iceberg offers, visit <a href="spec.html#partitioning">this page</a>.</p></div>
</div>
- <footer class="col-md-12 text-center">
-
- <hr>
- <p>
- <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.<br></small>
-
- <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</p></small>
+
+ <footer class="col-md-12 text-center">
+
+
+ <hr>
+ <p>
+ <small>Copyright 2018-2021 <a href='https://www.apache.org/'>The Apache Software Foundation</a><br />Apache Iceberg, Iceberg, Apache, the Apache feather logo, and the Apache Iceberg project logo are either registered<br />trademarks or trademarks of The Apache Software Foundation in the United States and other countries.</small><br>
+
+ <small>Documentation built with <a href="http://www.mkdocs.org/">MkDocs</a>.</small>
+ </p>
+
-
-
- </footer>
+
+
+ </footer>
+
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
- <script src="../js/bootstrap-3.0.3.min.js"></script>
- <script src="../js/highlight.pack.js"></script>
+ <script src="js/bootstrap-3.0.3.min.js"></script>
+
+
+ <script src="//cdn.jsdelivr.net/gh/highlightjs/cdn-release@9.18.0/build/highlight.min.js"></script>
+
<script>hljs.initHighlightingOnLoad();</script>
- <script>var base_url = ".."</script>
- <script src="../js/base.js"></script>
+
+ <script>var base_url = "."</script>
+
+ <script src="js/base.js"></script>
<div class="modal" id="mkdocs_keyboard_modal" tabindex="-1" role="dialog" aria-labelledby="keyboardModalLabel" aria-hidden="true">
<div class="modal-dialog">
diff --git a/javadoc/0.10.0/org/apache/iceberg/flink/FlinkTableFactory.html b/javadoc/0.10.0/org/apache/iceberg/flink/FlinkTableFactory.html
index 0128e5f..9d4579d 100644
--- a/javadoc/0.10.0/org/apache/iceberg/flink/FlinkTableFactory.html
+++ b/javadoc/0.10.0/org/apache/iceberg/flink/FlinkTableFactory.html
@@ -99,7 +99,7 @@ var activeTableTab = "activeTableTab";
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
-<li>org.apache.iceberg.flink.FlinkTableFactory</li>
+<li>org.apache.iceberg.flink.FlinkDynamicTableFactory</li>
</ul>
</li>
</ul>
diff --git a/javadoc/0.11.0/org/apache/iceberg/flink/FlinkTableFactory.html b/javadoc/0.11.0/org/apache/iceberg/flink/FlinkTableFactory.html
index 6a9407a..b0b7b7f 100644
--- a/javadoc/0.11.0/org/apache/iceberg/flink/FlinkTableFactory.html
+++ b/javadoc/0.11.0/org/apache/iceberg/flink/FlinkTableFactory.html
@@ -99,7 +99,7 @@ var activeTableTab = "activeTableTab";
<li>java.lang.Object</li>
<li>
<ul class="inheritance">
-<li>org.apache.iceberg.flink.FlinkTableFactory</li>
+<li>org.apache.iceberg.flink.FlinkDynamicTableFactory</li>
</ul>
</li>
</ul>
diff --git a/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils-ie.js b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils-ie.js
new file mode 100644
index 0000000..a74cc70
--- /dev/null
+++ b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils-ie.js
@@ -0,0 +1,56 @@
+/*!
+
+JSZipUtils - A collection of cross-browser utilities to go along with JSZip.
+<http://stuk.github.io/jszip-utils>
+
+(c) 2014 Stuart Knightley, David Duponchel
+Dual licenced under the MIT license or GPLv3. See https://raw.github.com/Stuk/jszip-utils/master/LICENSE.markdown.
+
+*/
+;(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o<r.length;o++)s(r[o]);return s})({1:[function(require,module,exports){
+var global=typeof self !== "undefined" ? self : typeof window !== "undefined" ? window : {};/* jshint evil: true, newcap: false */
+/* global IEBinaryToArray_ByteStr, IEBinaryToArray_ByteStr_Last */
+"use strict";
+
+// Adapted from http://stackoverflow.com/questions/1095102/how-do-i-load-binary-image-data-using-javascript-and-xmlhttprequest
+var IEBinaryToArray_ByteStr_Script =
+ "<!-- IEBinaryToArray_ByteStr -->\r\n"+
+ "<script type='text/vbscript'>\r\n"+
+ "Function IEBinaryToArray_ByteStr(Binary)\r\n"+
+ " IEBinaryToArray_ByteStr = CStr(Binary)\r\n"+
+ "End Function\r\n"+
+ "Function IEBinaryToArray_ByteStr_Last(Binary)\r\n"+
+ " Dim lastIndex\r\n"+
+ " lastIndex = LenB(Binary)\r\n"+
+ " if lastIndex mod 2 Then\r\n"+
+ " IEBinaryToArray_ByteStr_Last = Chr( AscB( MidB( Binary, lastIndex, 1 ) ) )\r\n"+
+ " Else\r\n"+
+ " IEBinaryToArray_ByteStr_Last = "+'""'+"\r\n"+
+ " End If\r\n"+
+ "End Function\r\n"+
+ "</script>\r\n";
+
+// inject VBScript
+document.write(IEBinaryToArray_ByteStr_Script);
+
+global.JSZipUtils._getBinaryFromXHR = function (xhr) {
+ var binary = xhr.responseBody;
+ var byteMapping = {};
+ for ( var i = 0; i < 256; i++ ) {
+ for ( var j = 0; j < 256; j++ ) {
+ byteMapping[ String.fromCharCode( i + (j << 8) ) ] =
+ String.fromCharCode(i) + String.fromCharCode(j);
+ }
+ }
+ var rawBytes = IEBinaryToArray_ByteStr(binary);
+ var lastChr = IEBinaryToArray_ByteStr_Last(binary);
+ return rawBytes.replace(/[\s\S]/g, function( match ) {
+ return byteMapping[match];
+ }) + lastChr;
+};
+
+// enforcing Stuk's coding style
+// vim: set shiftwidth=4 softtabstop=4:
+
+},{}]},{},[1])
+;
diff --git a/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils-ie.min.js b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils-ie.min.js
new file mode 100644
index 0000000..93d8bc8
--- /dev/null
+++ b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils-ie.min.js
@@ -0,0 +1,10 @@
+/*!
+
+JSZipUtils - A collection of cross-browser utilities to go along with JSZip.
+<http://stuk.github.io/jszip-utils>
+
+(c) 2014 Stuart Knightley, David Duponchel
+Dual licenced under the MIT license or GPLv3. See https://raw.github.com/Stuk/jszip-utils/master/LICENSE.markdown.
+
+*/
+!function a(b,c,d){function e(g,h){if(!c[g]){if(!b[g]){var i="function"==typeof require&&require;if(!h&&i)return i(g,!0);if(f)return f(g,!0);throw new Error("Cannot find module '"+g+"'")}var j=c[g]={exports:{}};b[g][0].call(j.exports,function(a){var c=b[g][1][a];return e(c?c:a)},j,j.exports,a,b,c,d)}return c[g].exports}for(var f="function"==typeof require&&require,g=0;g<d.length;g++)e(d[g]);return e}({1:[function(){var a="undefined"!=typeof self?self:"undefined"!=typeof window?window:{}, [...]
diff --git a/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils.js b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils.js
new file mode 100644
index 0000000..775895e
--- /dev/null
+++ b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils.js
@@ -0,0 +1,118 @@
+/*!
+
+JSZipUtils - A collection of cross-browser utilities to go along with JSZip.
+<http://stuk.github.io/jszip-utils>
+
+(c) 2014 Stuart Knightley, David Duponchel
+Dual licenced under the MIT license or GPLv3. See https://raw.github.com/Stuk/jszip-utils/master/LICENSE.markdown.
+
+*/
+!function(e){"object"==typeof exports?module.exports=e():"function"==typeof define&&define.amd?define(e):"undefined"!=typeof window?window.JSZipUtils=e():"undefined"!=typeof global?global.JSZipUtils=e():"undefined"!=typeof self&&(self.JSZipUtils=e())}(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o [...]
+'use strict';
+
+var JSZipUtils = {};
+// just use the responseText with xhr1, response with xhr2.
+// The transformation doesn't throw away high-order byte (with responseText)
+// because JSZip handles that case. If not used with JSZip, you may need to
+// do it, see https://developer.mozilla.org/En/Using_XMLHttpRequest#Handling_binary_data
+JSZipUtils._getBinaryFromXHR = function (xhr) {
+ // for xhr.responseText, the 0xFF mask is applied by JSZip
+ return xhr.response || xhr.responseText;
+};
+
+// taken from jQuery
+function createStandardXHR() {
+ try {
+ return new window.XMLHttpRequest();
+ } catch( e ) {}
+}
+
+function createActiveXHR() {
+ try {
+ return new window.ActiveXObject("Microsoft.XMLHTTP");
+ } catch( e ) {}
+}
+
+// Create the request object
+var createXHR = window.ActiveXObject ?
+ /* Microsoft failed to properly
+ * implement the XMLHttpRequest in IE7 (can't request local files),
+ * so we use the ActiveXObject when it is available
+ * Additionally XMLHttpRequest can be disabled in IE7/IE8 so
+ * we need a fallback.
+ */
+ function() {
+ return createStandardXHR() || createActiveXHR();
+} :
+ // For all other browsers, use the standard XMLHttpRequest object
+ createStandardXHR;
+
+
+
+JSZipUtils.getBinaryContent = function(path, callback) {
+ /*
+ * Here is the tricky part : getting the data.
+ * In firefox/chrome/opera/... setting the mimeType to 'text/plain; charset=x-user-defined'
+ * is enough, the result is in the standard xhr.responseText.
+ * cf https://developer.mozilla.org/En/XMLHttpRequest/Using_XMLHttpRequest#Receiving_binary_data_in_older_browsers
+ * In IE <= 9, we must use (the IE only) attribute responseBody
+ * (for binary data, its content is different from responseText).
+ * In IE 10, the 'charset=x-user-defined' trick doesn't work, only the
+ * responseType will work :
+ * http://msdn.microsoft.com/en-us/library/ie/hh673569%28v=vs.85%29.aspx#Binary_Object_upload_and_download
+ *
+ * I'd like to use jQuery to avoid this XHR madness, but it doesn't support
+ * the responseType attribute : http://bugs.jquery.com/ticket/11461
+ */
+ try {
+
+ var xhr = createXHR();
+
+ xhr.open('GET', path, true);
+
+ // recent browsers
+ if ("responseType" in xhr) {
+ xhr.responseType = "arraybuffer";
+ }
+
+ // older browser
+ if(xhr.overrideMimeType) {
+ xhr.overrideMimeType("text/plain; charset=x-user-defined");
+ }
+
+ xhr.onreadystatechange = function(evt) {
+ var file, err;
+ // use `xhr` and not `this`... thanks IE
+ if (xhr.readyState === 4) {
+ if (xhr.status === 200 || xhr.status === 0) {
+ file = null;
+ err = null;
+ try {
+ file = JSZipUtils._getBinaryFromXHR(xhr);
+ } catch(e) {
+ err = new Error(e);
+ }
+ callback(err, file);
+ } else {
+ callback(new Error("Ajax error for " + path + " : " + this.status + " " + this.statusText), null);
+ }
+ }
+ };
+
+ xhr.send();
+
+ } catch (e) {
+ callback(new Error(e), null);
+ }
+};
+
+// export
+module.exports = JSZipUtils;
+
+// enforcing Stuk's coding style
+// vim: set shiftwidth=4 softtabstop=4:
+
+},{}]},{},[1])
+(1)
+});
+;
diff --git a/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils.min.js b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils.min.js
new file mode 100644
index 0000000..78c7f5e
--- /dev/null
+++ b/javadoc/0.9.0/jquery/jszip-utils/dist/jszip-utils.min.js
@@ -0,0 +1,10 @@
+/*!
+
+JSZipUtils - A collection of cross-browser utilities to go along with JSZip.
+<http://stuk.github.io/jszip-utils>
+
+(c) 2014 Stuart Knightley, David Duponchel
+Dual licenced under the MIT license or GPLv3. See https://raw.github.com/Stuk/jszip-utils/master/LICENSE.markdown.
+
+*/
+!function(a){"object"==typeof exports?module.exports=a():"function"==typeof define&&define.amd?define(a):"undefined"!=typeof window?window.JSZipUtils=a():"undefined"!=typeof global?global.JSZipUtils=a():"undefined"!=typeof self&&(self.JSZipUtils=a())}(function(){return function a(b,c,d){function e(g,h){if(!c[g]){if(!b[g]){var i="function"==typeof require&&require;if(!h&&i)return i(g,!0);if(f)return f(g,!0);throw new Error("Cannot find module '"+g+"'")}var j=c[g]={exports:{}};b[g][0].call [...]
diff --git a/javadoc/0.9.0/jquery/jszip/dist/jszip.js b/javadoc/0.9.0/jquery/jszip/dist/jszip.js
new file mode 100644
index 0000000..02804cd
--- /dev/null
+++ b/javadoc/0.9.0/jquery/jszip/dist/jszip.js
@@ -0,0 +1,11623 @@
+/*!
+
+JSZip v3.1.5 - A JavaScript class for generating and reading zip files
+<http://stuartk.com/jszip>
+
+(c) 2009-2016 Stuart Knightley <stuart [at] stuartk.com>
+Dual licenced under the MIT license or GPLv3. See https://raw.github.com/Stuk/jszip/master/LICENSE.markdown.
+
+JSZip uses the library pako released under the MIT license :
+https://github.com/nodeca/pako/blob/master/LICENSE
+*/
+
+(function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.JSZip = f()}})(function(){var define,module,exports;return (function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)r [...]
+'use strict';
+var utils = require('./utils');
+var support = require('./support');
+// private property
+var _keyStr = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
+
+
+// public method for encoding
+exports.encode = function(input) {
+ var output = [];
+ var chr1, chr2, chr3, enc1, enc2, enc3, enc4;
+ var i = 0, len = input.length, remainingBytes = len;
+
+ var isArray = utils.getTypeOf(input) !== "string";
+ while (i < input.length) {
+ remainingBytes = len - i;
+
+ if (!isArray) {
+ chr1 = input.charCodeAt(i++);
+ chr2 = i < len ? input.charCodeAt(i++) : 0;
+ chr3 = i < len ? input.charCodeAt(i++) : 0;
+ } else {
+ chr1 = input[i++];
+ chr2 = i < len ? input[i++] : 0;
+ chr3 = i < len ? input[i++] : 0;
+ }
+
+ enc1 = chr1 >> 2;
+ enc2 = ((chr1 & 3) << 4) | (chr2 >> 4);
+ enc3 = remainingBytes > 1 ? (((chr2 & 15) << 2) | (chr3 >> 6)) : 64;
+ enc4 = remainingBytes > 2 ? (chr3 & 63) : 64;
+
+ output.push(_keyStr.charAt(enc1) + _keyStr.charAt(enc2) + _keyStr.charAt(enc3) + _keyStr.charAt(enc4));
+
+ }
+
+ return output.join("");
+};
+
+// public method for decoding
+exports.decode = function(input) {
+ var chr1, chr2, chr3;
+ var enc1, enc2, enc3, enc4;
+ var i = 0, resultIndex = 0;
+
+ var dataUrlPrefix = "data:";
+
+ if (input.substr(0, dataUrlPrefix.length) === dataUrlPrefix) {
+ // This is a common error: people give a data url
+ // (data:image/png;base64,iVBOR...) with a {base64: true} and
+ // wonders why things don't work.
+ // We can detect that the string input looks like a data url but we
+ // *can't* be sure it is one: removing everything up to the comma would
+ // be too dangerous.
+ throw new Error("Invalid base64 input, it looks like a data url.");
+ }
+
+ input = input.replace(/[^A-Za-z0-9\+\/\=]/g, "");
+
+ var totalLength = input.length * 3 / 4;
+ if(input.charAt(input.length - 1) === _keyStr.charAt(64)) {
+ totalLength--;
+ }
+ if(input.charAt(input.length - 2) === _keyStr.charAt(64)) {
+ totalLength--;
+ }
+ if (totalLength % 1 !== 0) {
+ // totalLength is not an integer, the length does not match a valid
+ // base64 content. That can happen if:
+ // - the input is not a base64 content
+ // - the input is *almost* a base64 content, with a extra chars at the
+ // beginning or at the end
+ // - the input uses a base64 variant (base64url for example)
+ throw new Error("Invalid base64 input, bad content length.");
+ }
+ var output;
+ if (support.uint8array) {
+ output = new Uint8Array(totalLength|0);
+ } else {
+ output = new Array(totalLength|0);
+ }
+
+ while (i < input.length) {
+
+ enc1 = _keyStr.indexOf(input.charAt(i++));
+ enc2 = _keyStr.indexOf(input.charAt(i++));
+ enc3 = _keyStr.indexOf(input.charAt(i++));
+ enc4 = _keyStr.indexOf(input.charAt(i++));
+
+ chr1 = (enc1 << 2) | (enc2 >> 4);
+ chr2 = ((enc2 & 15) << 4) | (enc3 >> 2);
+ chr3 = ((enc3 & 3) << 6) | enc4;
+
+ output[resultIndex++] = chr1;
+
+ if (enc3 !== 64) {
+ output[resultIndex++] = chr2;
+ }
+ if (enc4 !== 64) {
+ output[resultIndex++] = chr3;
+ }
+
+ }
+
+ return output;
+};
+
+},{"./support":30,"./utils":32}],2:[function(require,module,exports){
+'use strict';
+
+var external = require("./external");
+var DataWorker = require('./stream/DataWorker');
+var DataLengthProbe = require('./stream/DataLengthProbe');
+var Crc32Probe = require('./stream/Crc32Probe');
+var DataLengthProbe = require('./stream/DataLengthProbe');
+
+/**
+ * Represent a compressed object, with everything needed to decompress it.
+ * @constructor
+ * @param {number} compressedSize the size of the data compressed.
+ * @param {number} uncompressedSize the size of the data after decompression.
+ * @param {number} crc32 the crc32 of the decompressed file.
+ * @param {object} compression the type of compression, see lib/compressions.js.
+ * @param {String|ArrayBuffer|Uint8Array|Buffer} data the compressed data.
+ */
+function CompressedObject(compressedSize, uncompressedSize, crc32, compression, data) {
+ this.compressedSize = compressedSize;
+ this.uncompressedSize = uncompressedSize;
+ this.crc32 = crc32;
+ this.compression = compression;
+ this.compressedContent = data;
+}
+
+CompressedObject.prototype = {
+ /**
+ * Create a worker to get the uncompressed content.
+ * @return {GenericWorker} the worker.
+ */
+ getContentWorker : function () {
+ var worker = new DataWorker(external.Promise.resolve(this.compressedContent))
+ .pipe(this.compression.uncompressWorker())
+ .pipe(new DataLengthProbe("data_length"));
+
+ var that = this;
+ worker.on("end", function () {
+ if(this.streamInfo['data_length'] !== that.uncompressedSize) {
+ throw new Error("Bug : uncompressed data size mismatch");
+ }
+ });
+ return worker;
+ },
+ /**
+ * Create a worker to get the compressed content.
+ * @return {GenericWorker} the worker.
+ */
+ getCompressedWorker : function () {
+ return new DataWorker(external.Promise.resolve(this.compressedContent))
+ .withStreamInfo("compressedSize", this.compressedSize)
+ .withStreamInfo("uncompressedSize", this.uncompressedSize)
+ .withStreamInfo("crc32", this.crc32)
+ .withStreamInfo("compression", this.compression)
+ ;
+ }
+};
+
+/**
+ * Chain the given worker with other workers to compress the content with the
+ * given compresion.
+ * @param {GenericWorker} uncompressedWorker the worker to pipe.
+ * @param {Object} compression the compression object.
+ * @param {Object} compressionOptions the options to use when compressing.
+ * @return {GenericWorker} the new worker compressing the content.
+ */
+CompressedObject.createWorkerFrom = function (uncompressedWorker, compression, compressionOptions) {
+ return uncompressedWorker
+ .pipe(new Crc32Probe())
+ .pipe(new DataLengthProbe("uncompressedSize"))
+ .pipe(compression.compressWorker(compressionOptions))
+ .pipe(new DataLengthProbe("compressedSize"))
+ .withStreamInfo("compression", compression);
+};
+
+module.exports = CompressedObject;
+
+},{"./external":6,"./stream/Crc32Probe":25,"./stream/DataLengthProbe":26,"./stream/DataWorker":27}],3:[function(require,module,exports){
+'use strict';
+
+var GenericWorker = require("./stream/GenericWorker");
+
+exports.STORE = {
+ magic: "\x00\x00",
+ compressWorker : function (compressionOptions) {
+ return new GenericWorker("STORE compression");
+ },
+ uncompressWorker : function () {
+ return new GenericWorker("STORE decompression");
+ }
+};
+exports.DEFLATE = require('./flate');
+
+},{"./flate":7,"./stream/GenericWorker":28}],4:[function(require,module,exports){
+'use strict';
+
+var utils = require('./utils');
+
+/**
+ * The following functions come from pako, from pako/lib/zlib/crc32.js
+ * released under the MIT license, see pako https://github.com/nodeca/pako/
+ */
+
+// Use ordinary array, since untyped makes no boost here
+function makeTable() {
+ var c, table = [];
+
+ for(var n =0; n < 256; n++){
+ c = n;
+ for(var k =0; k < 8; k++){
+ c = ((c&1) ? (0xEDB88320 ^ (c >>> 1)) : (c >>> 1));
+ }
+ table[n] = c;
+ }
+
+ return table;
+}
+
+// Create table on load. Just 255 signed longs. Not a problem.
+var crcTable = makeTable();
+
+
+function crc32(crc, buf, len, pos) {
+ var t = crcTable, end = pos + len;
+
+ crc = crc ^ (-1);
+
+ for (var i = pos; i < end; i++ ) {
+ crc = (crc >>> 8) ^ t[(crc ^ buf[i]) & 0xFF];
+ }
+
+ return (crc ^ (-1)); // >>> 0;
+}
+
+// That's all for the pako functions.
+
+/**
+ * Compute the crc32 of a string.
+ * This is almost the same as the function crc32, but for strings. Using the
+ * same function for the two use cases leads to horrible performances.
+ * @param {Number} crc the starting value of the crc.
+ * @param {String} str the string to use.
+ * @param {Number} len the length of the string.
+ * @param {Number} pos the starting position for the crc32 computation.
+ * @return {Number} the computed crc32.
+ */
+function crc32str(crc, str, len, pos) {
+ var t = crcTable, end = pos + len;
+
+ crc = crc ^ (-1);
+
+ for (var i = pos; i < end; i++ ) {
+ crc = (crc >>> 8) ^ t[(crc ^ str.charCodeAt(i)) & 0xFF];
+ }
+
+ return (crc ^ (-1)); // >>> 0;
+}
+
+module.exports = function crc32wrapper(input, crc) {
+ if (typeof input === "undefined" || !input.length) {
+ return 0;
+ }
+
+ var isArray = utils.getTypeOf(input) !== "string";
+
+ if(isArray) {
+ return crc32(crc|0, input, input.length, 0);
+ } else {
+ return crc32str(crc|0, input, input.length, 0);
+ }
+};
+
+},{"./utils":32}],5:[function(require,module,exports){
+'use strict';
+exports.base64 = false;
+exports.binary = false;
+exports.dir = false;
+exports.createFolders = true;
+exports.date = null;
+exports.compression = null;
+exports.compressionOptions = null;
+exports.comment = null;
+exports.unixPermissions = null;
+exports.dosPermissions = null;
+
+},{}],6:[function(require,module,exports){
+/* global Promise */
+'use strict';
+
+// load the global object first:
+// - it should be better integrated in the system (unhandledRejection in node)
+// - the environment may have a custom Promise implementation (see zone.js)
+var ES6Promise = null;
+if (typeof Promise !== "undefined") {
+ ES6Promise = Promise;
+} else {
+ ES6Promise = require("lie");
+}
+
+/**
+ * Let the user use/change some implementations.
+ */
+module.exports = {
+ Promise: ES6Promise
+};
+
+},{"lie":58}],7:[function(require,module,exports){
+'use strict';
+var USE_TYPEDARRAY = (typeof Uint8Array !== 'undefined') && (typeof Uint16Array !== 'undefined') && (typeof Uint32Array !== 'undefined');
+
+var pako = require("pako");
+var utils = require("./utils");
+var GenericWorker = require("./stream/GenericWorker");
+
+var ARRAY_TYPE = USE_TYPEDARRAY ? "uint8array" : "array";
+
+exports.magic = "\x08\x00";
+
+/**
+ * Create a worker that uses pako to inflate/deflate.
+ * @constructor
+ * @param {String} action the name of the pako function to call : either "Deflate" or "Inflate".
+ * @param {Object} options the options to use when (de)compressing.
+ */
+function FlateWorker(action, options) {
+ GenericWorker.call(this, "FlateWorker/" + action);
+
+ this._pako = null;
+ this._pakoAction = action;
+ this._pakoOptions = options;
+ // the `meta` object from the last chunk received
+ // this allow this worker to pass around metadata
+ this.meta = {};
+}
+
+utils.inherits(FlateWorker, GenericWorker);
+
+/**
+ * @see GenericWorker.processChunk
+ */
+FlateWorker.prototype.processChunk = function (chunk) {
+ this.meta = chunk.meta;
+ if (this._pako === null) {
+ this._createPako();
+ }
+ this._pako.push(utils.transformTo(ARRAY_TYPE, chunk.data), false);
+};
+
+/**
+ * @see GenericWorker.flush
+ */
+FlateWorker.prototype.flush = function () {
+ GenericWorker.prototype.flush.call(this);
+ if (this._pako === null) {
+ this._createPako();
+ }
+ this._pako.push([], true);
+};
+/**
+ * @see GenericWorker.cleanUp
+ */
+FlateWorker.prototype.cleanUp = function () {
+ GenericWorker.prototype.cleanUp.call(this);
+ this._pako = null;
+};
+
+/**
+ * Create the _pako object.
+ * TODO: lazy-loading this object isn't the best solution but it's the
+ * quickest. The best solution is to lazy-load the worker list. See also the
+ * issue #446.
+ */
+FlateWorker.prototype._createPako = function () {
+ this._pako = new pako[this._pakoAction]({
+ raw: true,
+ level: this._pakoOptions.level || -1 // default compression
+ });
+ var self = this;
+ this._pako.onData = function(data) {
+ self.push({
+ data : data,
+ meta : self.meta
+ });
+ };
+};
+
+exports.compressWorker = function (compressionOptions) {
+ return new FlateWorker("Deflate", compressionOptions);
+};
+exports.uncompressWorker = function () {
+ return new FlateWorker("Inflate", {});
+};
+
+},{"./stream/GenericWorker":28,"./utils":32,"pako":59}],8:[function(require,module,exports){
+'use strict';
+
+var utils = require('../utils');
+var GenericWorker = require('../stream/GenericWorker');
+var utf8 = require('../utf8');
+var crc32 = require('../crc32');
+var signature = require('../signature');
+
+/**
+ * Transform an integer into a string in hexadecimal.
+ * @private
+ * @param {number} dec the number to convert.
+ * @param {number} bytes the number of bytes to generate.
+ * @returns {string} the result.
+ */
+var decToHex = function(dec, bytes) {
+ var hex = "", i;
+ for (i = 0; i < bytes; i++) {
+ hex += String.fromCharCode(dec & 0xff);
+ dec = dec >>> 8;
+ }
+ return hex;
+};
+
+/**
+ * Generate the UNIX part of the external file attributes.
+ * @param {Object} unixPermissions the unix permissions or null.
+ * @param {Boolean} isDir true if the entry is a directory, false otherwise.
+ * @return {Number} a 32 bit integer.
+ *
+ * adapted from http://unix.stackexchange.com/questions/14705/the-zip-formats-external-file-attribute :
+ *
+ * TTTTsstrwxrwxrwx0000000000ADVSHR
+ * ^^^^____________________________ file type, see zipinfo.c (UNX_*)
+ * ^^^_________________________ setuid, setgid, sticky
+ * ^^^^^^^^^________________ permissions
+ * ^^^^^^^^^^______ not used ?
+ * ^^^^^^ DOS attribute bits : Archive, Directory, Volume label, System file, Hidden, Read only
+ */
+var generateUnixExternalFileAttr = function (unixPermissions, isDir) {
+
+ var result = unixPermissions;
+ if (!unixPermissions) {
+ // I can't use octal values in strict mode, hence the hexa.
+ // 040775 => 0x41fd
+ // 0100664 => 0x81b4
+ result = isDir ? 0x41fd : 0x81b4;
+ }
+ return (result & 0xFFFF) << 16;
+};
+
+/**
+ * Generate the DOS part of the external file attributes.
+ * @param {Object} dosPermissions the dos permissions or null.
+ * @param {Boolean} isDir true if the entry is a directory, false otherwise.
+ * @return {Number} a 32 bit integer.
+ *
+ * Bit 0 Read-Only
+ * Bit 1 Hidden
+ * Bit 2 System
+ * Bit 3 Volume Label
+ * Bit 4 Directory
+ * Bit 5 Archive
+ */
+var generateDosExternalFileAttr = function (dosPermissions, isDir) {
+
+ // the dir flag is already set for compatibility
+ return (dosPermissions || 0) & 0x3F;
+};
+
+/**
+ * Generate the various parts used in the construction of the final zip file.
+ * @param {Object} streamInfo the hash with informations about the compressed file.
+ * @param {Boolean} streamedContent is the content streamed ?
+ * @param {Boolean} streamingEnded is the stream finished ?
+ * @param {number} offset the current offset from the start of the zip file.
+ * @param {String} platform let's pretend we are this platform (change platform dependents fields)
+ * @param {Function} encodeFileName the function to encode the file name / comment.
+ * @return {Object} the zip parts.
+ */
+var generateZipParts = function(streamInfo, streamedContent, streamingEnded, offset, platform, encodeFileName) {
+ var file = streamInfo['file'],
+ compression = streamInfo['compression'],
+ useCustomEncoding = encodeFileName !== utf8.utf8encode,
+ encodedFileName = utils.transformTo("string", encodeFileName(file.name)),
+ utfEncodedFileName = utils.transformTo("string", utf8.utf8encode(file.name)),
+ comment = file.comment,
+ encodedComment = utils.transformTo("string", encodeFileName(comment)),
+ utfEncodedComment = utils.transformTo("string", utf8.utf8encode(comment)),
+ useUTF8ForFileName = utfEncodedFileName.length !== file.name.length,
+ useUTF8ForComment = utfEncodedComment.length !== comment.length,
+ dosTime,
+ dosDate,
+ extraFields = "",
+ unicodePathExtraField = "",
+ unicodeCommentExtraField = "",
+ dir = file.dir,
+ date = file.date;
+
+
+ var dataInfo = {
+ crc32 : 0,
+ compressedSize : 0,
+ uncompressedSize : 0
+ };
+
+ // if the content is streamed, the sizes/crc32 are only available AFTER
+ // the end of the stream.
+ if (!streamedContent || streamingEnded) {
+ dataInfo.crc32 = streamInfo['crc32'];
+ dataInfo.compressedSize = streamInfo['compressedSize'];
+ dataInfo.uncompressedSize = streamInfo['uncompressedSize'];
+ }
+
+ var bitflag = 0;
+ if (streamedContent) {
+ // Bit 3: the sizes/crc32 are set to zero in the local header.
+ // The correct values are put in the data descriptor immediately
+ // following the compressed data.
+ bitflag |= 0x0008;
+ }
+ if (!useCustomEncoding && (useUTF8ForFileName || useUTF8ForComment)) {
+ // Bit 11: Language encoding flag (EFS).
+ bitflag |= 0x0800;
+ }
+
+
+ var extFileAttr = 0;
+ var versionMadeBy = 0;
+ if (dir) {
+ // dos or unix, we set the dos dir flag
+ extFileAttr |= 0x00010;
+ }
+ if(platform === "UNIX") {
+ versionMadeBy = 0x031E; // UNIX, version 3.0
+ extFileAttr |= generateUnixExternalFileAttr(file.unixPermissions, dir);
+ } else { // DOS or other, fallback to DOS
+ versionMadeBy = 0x0014; // DOS, version 2.0
+ extFileAttr |= generateDosExternalFileAttr(file.dosPermissions, dir);
+ }
+
+ // date
+ // @see http://www.delorie.com/djgpp/doc/rbinter/it/52/13.html
+ // @see http://www.delorie.com/djgpp/doc/rbinter/it/65/16.html
+ // @see http://www.delorie.com/djgpp/doc/rbinter/it/66/16.html
+
+ dosTime = date.getUTCHours();
+ dosTime = dosTime << 6;
+ dosTime = dosTime | date.getUTCMinutes();
+ dosTime = dosTime << 5;
+ dosTime = dosTime | date.getUTCSeconds() / 2;
+
+ dosDate = date.getUTCFullYear() - 1980;
+ dosDate = dosDate << 4;
+ dosDate = dosDate | (date.getUTCMonth() + 1);
+ dosDate = dosDate << 5;
+ dosDate = dosDate | date.getUTCDate();
+
+ if (useUTF8ForFileName) {
+ // set the unicode path extra field. unzip needs at least one extra
+ // field to correctly handle unicode path, so using the path is as good
+ // as any other information. This could improve the situation with
+ // other archive managers too.
+ // This field is usually used without the utf8 flag, with a non
+ // unicode path in the header (winrar, winzip). This helps (a bit)
+ // with the messy Windows' default compressed folders feature but
+ // breaks on p7zip which doesn't seek the unicode path extra field.
+ // So for now, UTF-8 everywhere !
+ unicodePathExtraField =
+ // Version
+ decToHex(1, 1) +
+ // NameCRC32
+ decToHex(crc32(encodedFileName), 4) +
+ // UnicodeName
+ utfEncodedFileName;
+
+ extraFields +=
+ // Info-ZIP Unicode Path Extra Field
+ "\x75\x70" +
+ // size
+ decToHex(unicodePathExtraField.length, 2) +
+ // content
+ unicodePathExtraField;
+ }
+
+ if(useUTF8ForComment) {
+
+ unicodeCommentExtraField =
+ // Version
+ decToHex(1, 1) +
+ // CommentCRC32
+ decToHex(crc32(encodedComment), 4) +
+ // UnicodeName
+ utfEncodedComment;
+
+ extraFields +=
+ // Info-ZIP Unicode Path Extra Field
+ "\x75\x63" +
+ // size
+ decToHex(unicodeCommentExtraField.length, 2) +
+ // content
+ unicodeCommentExtraField;
+ }
+
+ var header = "";
+
+ // version needed to extract
+ header += "\x0A\x00";
+ // general purpose bit flag
+ header += decToHex(bitflag, 2);
+ // compression method
+ header += compression.magic;
+ // last mod file time
+ header += decToHex(dosTime, 2);
+ // last mod file date
+ header += decToHex(dosDate, 2);
+ // crc-32
+ header += decToHex(dataInfo.crc32, 4);
+ // compressed size
+ header += decToHex(dataInfo.compressedSize, 4);
+ // uncompressed size
+ header += decToHex(dataInfo.uncompressedSize, 4);
+ // file name length
+ header += decToHex(encodedFileName.length, 2);
+ // extra field length
+ header += decToHex(extraFields.length, 2);
+
+
+ var fileRecord = signature.LOCAL_FILE_HEADER + header + encodedFileName + extraFields;
+
+ var dirRecord = signature.CENTRAL_FILE_HEADER +
+ // version made by (00: DOS)
+ decToHex(versionMadeBy, 2) +
+ // file header (common to file and central directory)
+ header +
+ // file comment length
+ decToHex(encodedComment.length, 2) +
+ // disk number start
+ "\x00\x00" +
+ // internal file attributes TODO
+ "\x00\x00" +
+ // external file attributes
+ decToHex(extFileAttr, 4) +
+ // relative offset of local header
+ decToHex(offset, 4) +
+ // file name
+ encodedFileName +
+ // extra field
+ extraFields +
+ // file comment
+ encodedComment;
+
+ return {
+ fileRecord: fileRecord,
+ dirRecord: dirRecord
+ };
+};
+
+/**
+ * Generate the EOCD record.
+ * @param {Number} entriesCount the number of entries in the zip file.
+ * @param {Number} centralDirLength the length (in bytes) of the central dir.
+ * @param {Number} localDirLength the length (in bytes) of the local dir.
+ * @param {String} comment the zip file comment as a binary string.
+ * @param {Function} encodeFileName the function to encode the comment.
+ * @return {String} the EOCD record.
+ */
+var generateCentralDirectoryEnd = function (entriesCount, centralDirLength, localDirLength, comment, encodeFileName) {
+ var dirEnd = "";
+ var encodedComment = utils.transformTo("string", encodeFileName(comment));
+
+ // end of central dir signature
+ dirEnd = signature.CENTRAL_DIRECTORY_END +
+ // number of this disk
+ "\x00\x00" +
+ // number of the disk with the start of the central directory
+ "\x00\x00" +
+ // total number of entries in the central directory on this disk
+ decToHex(entriesCount, 2) +
+ // total number of entries in the central directory
+ decToHex(entriesCount, 2) +
+ // size of the central directory 4 bytes
+ decToHex(centralDirLength, 4) +
+ // offset of start of central directory with respect to the starting disk number
+ decToHex(localDirLength, 4) +
+ // .ZIP file comment length
+ decToHex(encodedComment.length, 2) +
+ // .ZIP file comment
+ encodedComment;
+
+ return dirEnd;
+};
+
+/**
+ * Generate data descriptors for a file entry.
+ * @param {Object} streamInfo the hash generated by a worker, containing informations
+ * on the file entry.
+ * @return {String} the data descriptors.
+ */
+var generateDataDescriptors = function (streamInfo) {
+ var descriptor = "";
+ descriptor = signature.DATA_DESCRIPTOR +
+ // crc-32 4 bytes
+ decToHex(streamInfo['crc32'], 4) +
+ // compressed size 4 bytes
+ decToHex(streamInfo['compressedSize'], 4) +
+ // uncompressed size 4 bytes
+ decToHex(streamInfo['uncompressedSize'], 4);
+
+ return descriptor;
+};
+
+
+/**
+ * A worker to concatenate other workers to create a zip file.
+ * @param {Boolean} streamFiles `true` to stream the content of the files,
+ * `false` to accumulate it.
+ * @param {String} comment the comment to use.
+ * @param {String} platform the platform to use, "UNIX" or "DOS".
+ * @param {Function} encodeFileName the function to encode file names and comments.
+ */
+function ZipFileWorker(streamFiles, comment, platform, encodeFileName) {
+ GenericWorker.call(this, "ZipFileWorker");
+ // The number of bytes written so far. This doesn't count accumulated chunks.
+ this.bytesWritten = 0;
+ // The comment of the zip file
+ this.zipComment = comment;
+ // The platform "generating" the zip file.
+ this.zipPlatform = platform;
+ // the function to encode file names and comments.
+ this.encodeFileName = encodeFileName;
+ // Should we stream the content of the files ?
+ this.streamFiles = streamFiles;
+ // If `streamFiles` is false, we will need to accumulate the content of the
+ // files to calculate sizes / crc32 (and write them *before* the content).
+ // This boolean indicates if we are accumulating chunks (it will change a lot
+ // during the lifetime of this worker).
+ this.accumulate = false;
+ // The buffer receiving chunks when accumulating content.
+ this.contentBuffer = [];
+ // The list of generated directory records.
+ this.dirRecords = [];
+ // The offset (in bytes) from the beginning of the zip file for the current source.
+ this.currentSourceOffset = 0;
+ // The total number of entries in this zip file.
+ this.entriesCount = 0;
+ // the name of the file currently being added, null when handling the end of the zip file.
+ // Used for the emited metadata.
+ this.currentFile = null;
+
+
+
+ this._sources = [];
+}
+utils.inherits(ZipFileWorker, GenericWorker);
+
+/**
+ * @see GenericWorker.push
+ */
+ZipFileWorker.prototype.push = function (chunk) {
+
+ var currentFilePercent = chunk.meta.percent || 0;
+ var entriesCount = this.entriesCount;
+ var remainingFiles = this._sources.length;
+
+ if(this.accumulate) {
+ this.contentBuffer.push(chunk);
+ } else {
+ this.bytesWritten += chunk.data.length;
+
+ GenericWorker.prototype.push.call(this, {
+ data : chunk.data,
+ meta : {
+ currentFile : this.currentFile,
+ percent : entriesCount ? (currentFilePercent + 100 * (entriesCount - remainingFiles - 1)) / entriesCount : 100
+ }
+ });
+ }
+};
+
+/**
+ * The worker started a new source (an other worker).
+ * @param {Object} streamInfo the streamInfo object from the new source.
+ */
+ZipFileWorker.prototype.openedSource = function (streamInfo) {
+ this.currentSourceOffset = this.bytesWritten;
+ this.currentFile = streamInfo['file'].name;
+
+ var streamedContent = this.streamFiles && !streamInfo['file'].dir;
+
+ // don't stream folders (because they don't have any content)
+ if(streamedContent) {
+ var record = generateZipParts(streamInfo, streamedContent, false, this.currentSourceOffset, this.zipPlatform, this.encodeFileName);
+ this.push({
+ data : record.fileRecord,
+ meta : {percent:0}
+ });
+ } else {
+ // we need to wait for the whole file before pushing anything
+ this.accumulate = true;
+ }
+};
+
+/**
+ * The worker finished a source (an other worker).
+ * @param {Object} streamInfo the streamInfo object from the finished source.
+ */
+ZipFileWorker.prototype.closedSource = function (streamInfo) {
+ this.accumulate = false;
+ var streamedContent = this.streamFiles && !streamInfo['file'].dir;
+ var record = generateZipParts(streamInfo, streamedContent, true, this.currentSourceOffset, this.zipPlatform, this.encodeFileName);
+
+ this.dirRecords.push(record.dirRecord);
+ if(streamedContent) {
+ // after the streamed file, we put data descriptors
+ this.push({
+ data : generateDataDescriptors(streamInfo),
+ meta : {percent:100}
+ });
+ } else {
+ // the content wasn't streamed, we need to push everything now
+ // first the file record, then the content
+ this.push({
+ data : record.fileRecord,
+ meta : {percent:0}
+ });
+ while(this.contentBuffer.length) {
+ this.push(this.contentBuffer.shift());
+ }
+ }
+ this.currentFile = null;
+};
+
+/**
+ * @see GenericWorker.flush
+ */
+ZipFileWorker.prototype.flush = function () {
+
+ var localDirLength = this.bytesWritten;
+ for(var i = 0; i < this.dirRecords.length; i++) {
+ this.push({
+ data : this.dirRecords[i],
+ meta : {percent:100}
+ });
+ }
+ var centralDirLength = this.bytesWritten - localDirLength;
+
+ var dirEnd = generateCentralDirectoryEnd(this.dirRecords.length, centralDirLength, localDirLength, this.zipComment, this.encodeFileName);
+
+ this.push({
+ data : dirEnd,
+ meta : {percent:100}
+ });
+};
+
+/**
+ * Prepare the next source to be read.
+ */
+ZipFileWorker.prototype.prepareNextSource = function () {
+ this.previous = this._sources.shift();
+ this.openedSource(this.previous.streamInfo);
+ if (this.isPaused) {
+ this.previous.pause();
+ } else {
+ this.previous.resume();
+ }
+};
+
+/**
+ * @see GenericWorker.registerPrevious
+ */
+ZipFileWorker.prototype.registerPrevious = function (previous) {
+ this._sources.push(previous);
+ var self = this;
+
+ previous.on('data', function (chunk) {
+ self.processChunk(chunk);
+ });
+ previous.on('end', function () {
+ self.closedSource(self.previous.streamInfo);
+ if(self._sources.length) {
+ self.prepareNextSource();
+ } else {
+ self.end();
+ }
+ });
+ previous.on('error', function (e) {
+ self.error(e);
+ });
+ return this;
+};
+
+/**
+ * @see GenericWorker.resume
+ */
+ZipFileWorker.prototype.resume = function () {
+ if(!GenericWorker.prototype.resume.call(this)) {
+ return false;
+ }
+
+ if (!this.previous && this._sources.length) {
+ this.prepareNextSource();
+ return true;
+ }
+ if (!this.previous && !this._sources.length && !this.generatedError) {
+ this.end();
+ return true;
+ }
+};
+
+/**
+ * @see GenericWorker.error
+ */
+ZipFileWorker.prototype.error = function (e) {
+ var sources = this._sources;
+ if(!GenericWorker.prototype.error.call(this, e)) {
+ return false;
+ }
+ for(var i = 0; i < sources.length; i++) {
+ try {
+ sources[i].error(e);
+ } catch(e) {
+ // the `error` exploded, nothing to do
+ }
+ }
+ return true;
+};
+
+/**
+ * @see GenericWorker.lock
+ */
+ZipFileWorker.prototype.lock = function () {
+ GenericWorker.prototype.lock.call(this);
+ var sources = this._sources;
+ for(var i = 0; i < sources.length; i++) {
+ sources[i].lock();
+ }
+};
+
+module.exports = ZipFileWorker;
+
+},{"../crc32":4,"../signature":23,"../stream/GenericWorker":28,"../utf8":31,"../utils":32}],9:[function(require,module,exports){
+'use strict';
+
+var compressions = require('../compressions');
+var ZipFileWorker = require('./ZipFileWorker');
+
+/**
+ * Find the compression to use.
+ * @param {String} fileCompression the compression defined at the file level, if any.
+ * @param {String} zipCompression the compression defined at the load() level.
+ * @return {Object} the compression object to use.
+ */
+var getCompression = function (fileCompression, zipCompression) {
+
+ var compressionName = fileCompression || zipCompression;
+ var compression = compressions[compressionName];
+ if (!compression) {
+ throw new Error(compressionName + " is not a valid compression method !");
+ }
+ return compression;
+};
+
+/**
+ * Create a worker to generate a zip file.
+ * @param {JSZip} zip the JSZip instance at the right root level.
+ * @param {Object} options to generate the zip file.
+ * @param {String} comment the comment to use.
+ */
+exports.generateWorker = function (zip, options, comment) {
+
+ var zipFileWorker = new ZipFileWorker(options.streamFiles, comment, options.platform, options.encodeFileName);
+ var entriesCount = 0;
+ try {
+
+ zip.forEach(function (relativePath, file) {
+ entriesCount++;
+ var compression = getCompression(file.options.compression, options.compression);
+ var compressionOptions = file.options.compressionOptions || options.compressionOptions || {};
+ var dir = file.dir, date = file.date;
+
+ file._compressWorker(compression, compressionOptions)
+ .withStreamInfo("file", {
+ name : relativePath,
+ dir : dir,
+ date : date,
+ comment : file.comment || "",
+ unixPermissions : file.unixPermissions,
+ dosPermissions : file.dosPermissions
+ })
+ .pipe(zipFileWorker);
+ });
+ zipFileWorker.entriesCount = entriesCount;
+ } catch (e) {
+ zipFileWorker.error(e);
+ }
+
+ return zipFileWorker;
+};
+
+},{"../compressions":3,"./ZipFileWorker":8}],10:[function(require,module,exports){
+'use strict';
+
+/**
+ * Representation a of zip file in js
+ * @constructor
+ */
+function JSZip() {
+ // if this constructor is used without `new`, it adds `new` before itself:
+ if(!(this instanceof JSZip)) {
+ return new JSZip();
+ }
+
+ if(arguments.length) {
+ throw new Error("The constructor with parameters has been removed in JSZip 3.0, please check the upgrade guide.");
+ }
+
+ // object containing the files :
+ // {
+ // "folder/" : {...},
+ // "folder/data.txt" : {...}
+ // }
+ this.files = {};
+
+ this.comment = null;
+
+ // Where we are in the hierarchy
+ this.root = "";
+ this.clone = function() {
+ var newObj = new JSZip();
+ for (var i in this) {
+ if (typeof this[i] !== "function") {
+ newObj[i] = this[i];
+ }
+ }
+ return newObj;
+ };
+}
+JSZip.prototype = require('./object');
+JSZip.prototype.loadAsync = require('./load');
+JSZip.support = require('./support');
+JSZip.defaults = require('./defaults');
+
+// TODO find a better way to handle this version,
+// a require('package.json').version doesn't work with webpack, see #327
+JSZip.version = "3.1.5";
+
+JSZip.loadAsync = function (content, options) {
+ return new JSZip().loadAsync(content, options);
+};
+
+JSZip.external = require("./external");
+module.exports = JSZip;
+
+},{"./defaults":5,"./external":6,"./load":11,"./object":15,"./support":30}],11:[function(require,module,exports){
+'use strict';
+var utils = require('./utils');
+var external = require("./external");
+var utf8 = require('./utf8');
+var utils = require('./utils');
+var ZipEntries = require('./zipEntries');
+var Crc32Probe = require('./stream/Crc32Probe');
+var nodejsUtils = require("./nodejsUtils");
+
+/**
+ * Check the CRC32 of an entry.
+ * @param {ZipEntry} zipEntry the zip entry to check.
+ * @return {Promise} the result.
+ */
+function checkEntryCRC32(zipEntry) {
+ return new external.Promise(function (resolve, reject) {
+ var worker = zipEntry.decompressed.getContentWorker().pipe(new Crc32Probe());
+ worker.on("error", function (e) {
+ reject(e);
+ })
+ .on("end", function () {
+ if (worker.streamInfo.crc32 !== zipEntry.decompressed.crc32) {
+ reject(new Error("Corrupted zip : CRC32 mismatch"));
+ } else {
+ resolve();
+ }
+ })
+ .resume();
+ });
+}
+
+module.exports = function(data, options) {
+ var zip = this;
+ options = utils.extend(options || {}, {
+ base64: false,
+ checkCRC32: false,
+ optimizedBinaryString: false,
+ createFolders: false,
+ decodeFileName: utf8.utf8decode
+ });
+
+ if (nodejsUtils.isNode && nodejsUtils.isStream(data)) {
+ return external.Promise.reject(new Error("JSZip can't accept a stream when loading a zip file."));
+ }
+
+ return utils.prepareContent("the loaded zip file", data, true, options.optimizedBinaryString, options.base64)
+ .then(function(data) {
+ var zipEntries = new ZipEntries(options);
+ zipEntries.load(data);
+ return zipEntries;
+ }).then(function checkCRC32(zipEntries) {
+ var promises = [external.Promise.resolve(zipEntries)];
+ var files = zipEntries.files;
+ if (options.checkCRC32) {
+ for (var i = 0; i < files.length; i++) {
+ promises.push(checkEntryCRC32(files[i]));
+ }
+ }
+ return external.Promise.all(promises);
+ }).then(function addFiles(results) {
+ var zipEntries = results.shift();
+ var files = zipEntries.files;
+ for (var i = 0; i < files.length; i++) {
+ var input = files[i];
+ zip.file(input.fileNameStr, input.decompressed, {
+ binary: true,
+ optimizedBinaryString: true,
+ date: input.date,
+ dir: input.dir,
+ comment : input.fileCommentStr.length ? input.fileCommentStr : null,
+ unixPermissions : input.unixPermissions,
+ dosPermissions : input.dosPermissions,
+ createFolders: options.createFolders
+ });
+ }
+ if (zipEntries.zipComment.length) {
+ zip.comment = zipEntries.zipComment;
+ }
+
+ return zip;
+ });
+};
+
+},{"./external":6,"./nodejsUtils":14,"./stream/Crc32Probe":25,"./utf8":31,"./utils":32,"./zipEntries":33}],12:[function(require,module,exports){
+"use strict";
+
+var utils = require('../utils');
+var GenericWorker = require('../stream/GenericWorker');
+
+/**
+ * A worker that use a nodejs stream as source.
+ * @constructor
+ * @param {String} filename the name of the file entry for this stream.
+ * @param {Readable} stream the nodejs stream.
+ */
+function NodejsStreamInputAdapter(filename, stream) {
+ GenericWorker.call(this, "Nodejs stream input adapter for " + filename);
+ this._upstreamEnded = false;
+ this._bindStream(stream);
+}
+
+utils.inherits(NodejsStreamInputAdapter, GenericWorker);
+
+/**
+ * Prepare the stream and bind the callbacks on it.
+ * Do this ASAP on node 0.10 ! A lazy binding doesn't always work.
+ * @param {Stream} stream the nodejs stream to use.
+ */
+NodejsStreamInputAdapter.prototype._bindStream = function (stream) {
+ var self = this;
+ this._stream = stream;
+ stream.pause();
+ stream
+ .on("data", function (chunk) {
+ self.push({
+ data: chunk,
+ meta : {
+ percent : 0
+ }
+ });
+ })
+ .on("error", function (e) {
+ if(self.isPaused) {
+ this.generatedError = e;
+ } else {
+ self.error(e);
+ }
+ })
+ .on("end", function () {
+ if(self.isPaused) {
+ self._upstreamEnded = true;
+ } else {
+ self.end();
+ }
+ });
+};
+NodejsStreamInputAdapter.prototype.pause = function () {
+ if(!GenericWorker.prototype.pause.call(this)) {
+ return false;
+ }
+ this._stream.pause();
+ return true;
+};
+NodejsStreamInputAdapter.prototype.resume = function () {
+ if(!GenericWorker.prototype.resume.call(this)) {
+ return false;
+ }
+
+ if(this._upstreamEnded) {
+ this.end();
+ } else {
+ this._stream.resume();
+ }
+
+ return true;
+};
+
+module.exports = NodejsStreamInputAdapter;
+
+},{"../stream/GenericWorker":28,"../utils":32}],13:[function(require,module,exports){
+'use strict';
+
+var Readable = require('readable-stream').Readable;
+
+var utils = require('../utils');
+utils.inherits(NodejsStreamOutputAdapter, Readable);
+
+/**
+* A nodejs stream using a worker as source.
+* @see the SourceWrapper in http://nodejs.org/api/stream.html
+* @constructor
+* @param {StreamHelper} helper the helper wrapping the worker
+* @param {Object} options the nodejs stream options
+* @param {Function} updateCb the update callback.
+*/
+function NodejsStreamOutputAdapter(helper, options, updateCb) {
+ Readable.call(this, options);
+ this._helper = helper;
+
+ var self = this;
+ helper.on("data", function (data, meta) {
+ if (!self.push(data)) {
+ self._helper.pause();
+ }
+ if(updateCb) {
+ updateCb(meta);
+ }
+ })
+ .on("error", function(e) {
+ self.emit('error', e);
+ })
+ .on("end", function () {
+ self.push(null);
+ });
+}
+
+
+NodejsStreamOutputAdapter.prototype._read = function() {
+ this._helper.resume();
+};
+
+module.exports = NodejsStreamOutputAdapter;
+
+},{"../utils":32,"readable-stream":16}],14:[function(require,module,exports){
+'use strict';
+
+module.exports = {
+ /**
+ * True if this is running in Nodejs, will be undefined in a browser.
+ * In a browser, browserify won't include this file and the whole module
+ * will be resolved an empty object.
+ */
+ isNode : typeof Buffer !== "undefined",
+ /**
+ * Create a new nodejs Buffer from an existing content.
+ * @param {Object} data the data to pass to the constructor.
+ * @param {String} encoding the encoding to use.
+ * @return {Buffer} a new Buffer.
+ */
+ newBufferFrom: function(data, encoding) {
+ // XXX We can't use `Buffer.from` which comes from `Uint8Array.from`
+ // in nodejs v4 (< v.4.5). It's not the expected implementation (and
+ // has a different signature).
+ // see https://github.com/nodejs/node/issues/8053
+ // A condition on nodejs' version won't solve the issue as we don't
+ // control the Buffer polyfills that may or may not be used.
+ return new Buffer(data, encoding);
+ },
+ /**
+ * Create a new nodejs Buffer with the specified size.
+ * @param {Integer} size the size of the buffer.
+ * @return {Buffer} a new Buffer.
+ */
+ allocBuffer: function (size) {
+ if (Buffer.alloc) {
+ return Buffer.alloc(size);
+ } else {
+ return new Buffer(size);
+ }
+ },
+ /**
+ * Find out if an object is a Buffer.
+ * @param {Object} b the object to test.
+ * @return {Boolean} true if the object is a Buffer, false otherwise.
+ */
+ isBuffer : function(b){
+ return Buffer.isBuffer(b);
+ },
+
+ isStream : function (obj) {
+ return obj &&
+ typeof obj.on === "function" &&
+ typeof obj.pause === "function" &&
+ typeof obj.resume === "function";
+ }
+};
+
+},{}],15:[function(require,module,exports){
+'use strict';
+var utf8 = require('./utf8');
+var utils = require('./utils');
+var GenericWorker = require('./stream/GenericWorker');
+var StreamHelper = require('./stream/StreamHelper');
+var defaults = require('./defaults');
+var CompressedObject = require('./compressedObject');
+var ZipObject = require('./zipObject');
+var generate = require("./generate");
+var nodejsUtils = require("./nodejsUtils");
+var NodejsStreamInputAdapter = require("./nodejs/NodejsStreamInputAdapter");
+
+
+/**
+ * Add a file in the current folder.
+ * @private
+ * @param {string} name the name of the file
+ * @param {String|ArrayBuffer|Uint8Array|Buffer} data the data of the file
+ * @param {Object} originalOptions the options of the file
+ * @return {Object} the new file.
+ */
+var fileAdd = function(name, data, originalOptions) {
+ // be sure sub folders exist
+ var dataType = utils.getTypeOf(data),
+ parent;
+
+
+ /*
+ * Correct options.
+ */
+
+ var o = utils.extend(originalOptions || {}, defaults);
+ o.date = o.date || new Date();
+ if (o.compression !== null) {
+ o.compression = o.compression.toUpperCase();
+ }
+
+ if (typeof o.unixPermissions === "string") {
+ o.unixPermissions = parseInt(o.unixPermissions, 8);
+ }
+
+ // UNX_IFDIR 0040000 see zipinfo.c
+ if (o.unixPermissions && (o.unixPermissions & 0x4000)) {
+ o.dir = true;
+ }
+ // Bit 4 Directory
+ if (o.dosPermissions && (o.dosPermissions & 0x0010)) {
+ o.dir = true;
+ }
+
+ if (o.dir) {
+ name = forceTrailingSlash(name);
+ }
+ if (o.createFolders && (parent = parentFolder(name))) {
+ folderAdd.call(this, parent, true);
+ }
+
+ var isUnicodeString = dataType === "string" && o.binary === false && o.base64 === false;
+ if (!originalOptions || typeof originalOptions.binary === "undefined") {
+ o.binary = !isUnicodeString;
+ }
+
+
+ var isCompressedEmpty = (data instanceof CompressedObject) && data.uncompressedSize === 0;
+
+ if (isCompressedEmpty || o.dir || !data || data.length === 0) {
+ o.base64 = false;
+ o.binary = true;
+ data = "";
+ o.compression = "STORE";
+ dataType = "string";
+ }
+
+ /*
+ * Convert content to fit.
+ */
+
+ var zipObjectContent = null;
+ if (data instanceof CompressedObject || data instanceof GenericWorker) {
+ zipObjectContent = data;
+ } else if (nodejsUtils.isNode && nodejsUtils.isStream(data)) {
+ zipObjectContent = new NodejsStreamInputAdapter(name, data);
+ } else {
+ zipObjectContent = utils.prepareContent(name, data, o.binary, o.optimizedBinaryString, o.base64);
+ }
+
+ var object = new ZipObject(name, zipObjectContent, o);
+ this.files[name] = object;
+ /*
+ TODO: we can't throw an exception because we have async promises
+ (we can have a promise of a Date() for example) but returning a
+ promise is useless because file(name, data) returns the JSZip
+ object for chaining. Should we break that to allow the user
+ to catch the error ?
+
+ return external.Promise.resolve(zipObjectContent)
+ .then(function () {
+ return object;
+ });
+ */
+};
+
+/**
+ * Find the parent folder of the path.
+ * @private
+ * @param {string} path the path to use
+ * @return {string} the parent folder, or ""
+ */
+var parentFolder = function (path) {
+ if (path.slice(-1) === '/') {
+ path = path.substring(0, path.length - 1);
+ }
+ var lastSlash = path.lastIndexOf('/');
+ return (lastSlash > 0) ? path.substring(0, lastSlash) : "";
+};
+
+/**
+ * Returns the path with a slash at the end.
+ * @private
+ * @param {String} path the path to check.
+ * @return {String} the path with a trailing slash.
+ */
+var forceTrailingSlash = function(path) {
+ // Check the name ends with a /
+ if (path.slice(-1) !== "/") {
+ path += "/"; // IE doesn't like substr(-1)
+ }
+ return path;
+};
+
+/**
+ * Add a (sub) folder in the current folder.
+ * @private
+ * @param {string} name the folder's name
+ * @param {boolean=} [createFolders] If true, automatically create sub
+ * folders. Defaults to false.
+ * @return {Object} the new folder.
+ */
+var folderAdd = function(name, createFolders) {
+ createFolders = (typeof createFolders !== 'undefined') ? createFolders : defaults.createFolders;
+
+ name = forceTrailingSlash(name);
+
+ // Does this folder already exist?
+ if (!this.files[name]) {
+ fileAdd.call(this, name, null, {
+ dir: true,
+ createFolders: createFolders
+ });
+ }
+ return this.files[name];
+};
+
+/**
+* Cross-window, cross-Node-context regular expression detection
+* @param {Object} object Anything
+* @return {Boolean} true if the object is a regular expression,
+* false otherwise
+*/
+function isRegExp(object) {
+ return Object.prototype.toString.call(object) === "[object RegExp]";
+}
+
+// return the actual prototype of JSZip
+var out = {
+ /**
+ * @see loadAsync
+ */
+ load: function() {
+ throw new Error("This method has been removed in JSZip 3.0, please check the upgrade guide.");
+ },
+
+
+ /**
+ * Call a callback function for each entry at this folder level.
+ * @param {Function} cb the callback function:
+ * function (relativePath, file) {...}
+ * It takes 2 arguments : the relative path and the file.
+ */
+ forEach: function(cb) {
+ var filename, relativePath, file;
+ for (filename in this.files) {
+ if (!this.files.hasOwnProperty(filename)) {
+ continue;
+ }
+ file = this.files[filename];
+ relativePath = filename.slice(this.root.length, filename.length);
+ if (relativePath && filename.slice(0, this.root.length) === this.root) { // the file is in the current root
+ cb(relativePath, file); // TODO reverse the parameters ? need to be clean AND consistent with the filter search fn...
+ }
+ }
+ },
+
+ /**
+ * Filter nested files/folders with the specified function.
+ * @param {Function} search the predicate to use :
+ * function (relativePath, file) {...}
+ * It takes 2 arguments : the relative path and the file.
+ * @return {Array} An array of matching elements.
+ */
+ filter: function(search) {
+ var result = [];
+ this.forEach(function (relativePath, entry) {
+ if (search(relativePath, entry)) { // the file matches the function
+ result.push(entry);
+ }
+
+ });
+ return result;
+ },
+
+ /**
+ * Add a file to the zip file, or search a file.
+ * @param {string|RegExp} name The name of the file to add (if data is defined),
+ * the name of the file to find (if no data) or a regex to match files.
+ * @param {String|ArrayBuffer|Uint8Array|Buffer} data The file data, either raw or base64 encoded
+ * @param {Object} o File options
+ * @return {JSZip|Object|Array} this JSZip object (when adding a file),
+ * a file (when searching by string) or an array of files (when searching by regex).
+ */
+ file: function(name, data, o) {
+ if (arguments.length === 1) {
+ if (isRegExp(name)) {
+ var regexp = name;
+ return this.filter(function(relativePath, file) {
+ return !file.dir && regexp.test(relativePath);
+ });
+ }
+ else { // text
+ var obj = this.files[this.root + name];
+ if (obj && !obj.dir) {
+ return obj;
+ } else {
+ return null;
+ }
+ }
+ }
+ else { // more than one argument : we have data !
+ name = this.root + name;
+ fileAdd.call(this, name, data, o);
+ }
+ return this;
+ },
+
+ /**
+ * Add a directory to the zip file, or search.
+ * @param {String|RegExp} arg The name of the directory to add, or a regex to search folders.
+ * @return {JSZip} an object with the new directory as the root, or an array containing matching folders.
+ */
+ folder: function(arg) {
+ if (!arg) {
+ return this;
+ }
+
+ if (isRegExp(arg)) {
+ return this.filter(function(relativePath, file) {
+ return file.dir && arg.test(relativePath);
+ });
+ }
+
+ // else, name is a new folder
+ var name = this.root + arg;
+ var newFolder = folderAdd.call(this, name);
+
+ // Allow chaining by returning a new object with this folder as the root
+ var ret = this.clone();
+ ret.root = newFolder.name;
+ return ret;
+ },
+
+ /**
+ * Delete a file, or a directory and all sub-files, from the zip
+ * @param {string} name the name of the file to delete
+ * @return {JSZip} this JSZip object
+ */
+ remove: function(name) {
+ name = this.root + name;
+ var file = this.files[name];
+ if (!file) {
+ // Look for any folders
+ if (name.slice(-1) !== "/") {
+ name += "/";
+ }
+ file = this.files[name];
+ }
+
+ if (file && !file.dir) {
+ // file
+ delete this.files[name];
+ } else {
+ // maybe a folder, delete recursively
+ var kids = this.filter(function(relativePath, file) {
+ return file.name.slice(0, name.length) === name;
+ });
+ for (var i = 0; i < kids.length; i++) {
+ delete this.files[kids[i].name];
+ }
+ }
+
+ return this;
+ },
+
+ /**
+ * Generate the complete zip file
+ * @param {Object} options the options to generate the zip file :
+ * - compression, "STORE" by default.
+ * - type, "base64" by default. Values are : string, base64, uint8array, arraybuffer, blob.
+ * @return {String|Uint8Array|ArrayBuffer|Buffer|Blob} the zip file
+ */
+ generate: function(options) {
+ throw new Error("This method has been removed in JSZip 3.0, please check the upgrade guide.");
+ },
+
+ /**
+ * Generate the complete zip file as an internal stream.
+ * @param {Object} options the options to generate the zip file :
+ * - compression, "STORE" by default.
+ * - type, "base64" by default. Values are : string, base64, uint8array, arraybuffer, blob.
+ * @return {StreamHelper} the streamed zip file.
+ */
+ generateInternalStream: function(options) {
+ var worker, opts = {};
+ try {
+ opts = utils.extend(options || {}, {
+ streamFiles: false,
+ compression: "STORE",
+ compressionOptions : null,
+ type: "",
+ platform: "DOS",
+ comment: null,
+ mimeType: 'application/zip',
+ encodeFileName: utf8.utf8encode
+ });
+
+ opts.type = opts.type.toLowerCase();
+ opts.compression = opts.compression.toUpperCase();
+
+ // "binarystring" is prefered but the internals use "string".
+ if(opts.type === "binarystring") {
+ opts.type = "string";
+ }
+
+ if (!opts.type) {
+ throw new Error("No output type specified.");
+ }
+
+ utils.checkSupport(opts.type);
+
+ // accept nodejs `process.platform`
+ if(
+ opts.platform === 'darwin' ||
+ opts.platform === 'freebsd' ||
+ opts.platform === 'linux' ||
+ opts.platform === 'sunos'
+ ) {
+ opts.platform = "UNIX";
+ }
+ if (opts.platform === 'win32') {
+ opts.platform = "DOS";
+ }
+
+ var comment = opts.comment || this.comment || "";
+ worker = generate.generateWorker(this, opts, comment);
+ } catch (e) {
+ worker = new GenericWorker("error");
+ worker.error(e);
+ }
+ return new StreamHelper(worker, opts.type || "string", opts.mimeType);
+ },
+ /**
+ * Generate the complete zip file asynchronously.
+ * @see generateInternalStream
+ */
+ generateAsync: function(options, onUpdate) {
+ return this.generateInternalStream(options).accumulate(onUpdate);
+ },
+ /**
+ * Generate the complete zip file asynchronously.
+ * @see generateInternalStream
+ */
+ generateNodeStream: function(options, onUpdate) {
+ options = options || {};
+ if (!options.type) {
+ options.type = "nodebuffer";
+ }
+ return this.generateInternalStream(options).toNodejsStream(onUpdate);
+ }
+};
+module.exports = out;
+
+},{"./compressedObject":2,"./defaults":5,"./generate":9,"./nodejs/NodejsStreamInputAdapter":12,"./nodejsUtils":14,"./stream/GenericWorker":28,"./stream/StreamHelper":29,"./utf8":31,"./utils":32,"./zipObject":35}],16:[function(require,module,exports){
+/*
+ * This file is used by module bundlers (browserify/webpack/etc) when
+ * including a stream implementation. We use "readable-stream" to get a
+ * consistent behavior between nodejs versions but bundlers often have a shim
+ * for "stream". Using this shim greatly improve the compatibility and greatly
+ * reduce the final size of the bundle (only one stream implementation, not
+ * two).
+ */
+module.exports = require("stream");
+
+},{"stream":undefined}],17:[function(require,module,exports){
+'use strict';
+var DataReader = require('./DataReader');
+var utils = require('../utils');
+
+function ArrayReader(data) {
+ DataReader.call(this, data);
+ for(var i = 0; i < this.data.length; i++) {
+ data[i] = data[i] & 0xFF;
+ }
+}
+utils.inherits(ArrayReader, DataReader);
+/**
+ * @see DataReader.byteAt
+ */
+ArrayReader.prototype.byteAt = function(i) {
+ return this.data[this.zero + i];
+};
+/**
+ * @see DataReader.lastIndexOfSignature
+ */
+ArrayReader.prototype.lastIndexOfSignature = function(sig) {
+ var sig0 = sig.charCodeAt(0),
+ sig1 = sig.charCodeAt(1),
+ sig2 = sig.charCodeAt(2),
+ sig3 = sig.charCodeAt(3);
+ for (var i = this.length - 4; i >= 0; --i) {
+ if (this.data[i] === sig0 && this.data[i + 1] === sig1 && this.data[i + 2] === sig2 && this.data[i + 3] === sig3) {
+ return i - this.zero;
+ }
+ }
+
+ return -1;
+};
+/**
+ * @see DataReader.readAndCheckSignature
+ */
+ArrayReader.prototype.readAndCheckSignature = function (sig) {
+ var sig0 = sig.charCodeAt(0),
+ sig1 = sig.charCodeAt(1),
+ sig2 = sig.charCodeAt(2),
+ sig3 = sig.charCodeAt(3),
+ data = this.readData(4);
+ return sig0 === data[0] && sig1 === data[1] && sig2 === data[2] && sig3 === data[3];
+};
+/**
+ * @see DataReader.readData
+ */
+ArrayReader.prototype.readData = function(size) {
+ this.checkOffset(size);
+ if(size === 0) {
+ return [];
+ }
+ var result = this.data.slice(this.zero + this.index, this.zero + this.index + size);
+ this.index += size;
+ return result;
+};
+module.exports = ArrayReader;
+
+},{"../utils":32,"./DataReader":18}],18:[function(require,module,exports){
+'use strict';
+var utils = require('../utils');
+
+function DataReader(data) {
+ this.data = data; // type : see implementation
+ this.length = data.length;
+ this.index = 0;
+ this.zero = 0;
+}
+DataReader.prototype = {
+ /**
+ * Check that the offset will not go too far.
+ * @param {string} offset the additional offset to check.
+ * @throws {Error} an Error if the offset is out of bounds.
+ */
+ checkOffset: function(offset) {
+ this.checkIndex(this.index + offset);
+ },
+ /**
+ * Check that the specified index will not be too far.
+ * @param {string} newIndex the index to check.
+ * @throws {Error} an Error if the index is out of bounds.
+ */
+ checkIndex: function(newIndex) {
+ if (this.length < this.zero + newIndex || newIndex < 0) {
+ throw new Error("End of data reached (data length = " + this.length + ", asked index = " + (newIndex) + "). Corrupted zip ?");
+ }
+ },
+ /**
+ * Change the index.
+ * @param {number} newIndex The new index.
+ * @throws {Error} if the new index is out of the data.
+ */
+ setIndex: function(newIndex) {
+ this.checkIndex(newIndex);
+ this.index = newIndex;
+ },
+ /**
+ * Skip the next n bytes.
+ * @param {number} n the number of bytes to skip.
+ * @throws {Error} if the new index is out of the data.
+ */
+ skip: function(n) {
+ this.setIndex(this.index + n);
+ },
+ /**
+ * Get the byte at the specified index.
+ * @param {number} i the index to use.
+ * @return {number} a byte.
+ */
+ byteAt: function(i) {
+ // see implementations
+ },
+ /**
+ * Get the next number with a given byte size.
+ * @param {number} size the number of bytes to read.
+ * @return {number} the corresponding number.
+ */
+ readInt: function(size) {
+ var result = 0,
+ i;
+ this.checkOffset(size);
+ for (i = this.index + size - 1; i >= this.index; i--) {
+ result = (result << 8) + this.byteAt(i);
+ }
+ this.index += size;
+ return result;
+ },
+ /**
+ * Get the next string with a given byte size.
+ * @param {number} size the number of bytes to read.
+ * @return {string} the corresponding string.
+ */
+ readString: function(size) {
+ return utils.transformTo("string", this.readData(size));
+ },
+ /**
+ * Get raw data without conversion, <size> bytes.
+ * @param {number} size the number of bytes to read.
+ * @return {Object} the raw data, implementation specific.
+ */
... 21849 lines suppressed ...