You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/08/26 00:53:26 UTC
[iceberg] branch asf-site updated: Deployed 6e1bd6370 with MkDocs
version: 1.0.4
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/asf-site by this push:
new a53b9b1 Deployed 6e1bd6370 with MkDocs version: 1.0.4
a53b9b1 is described below
commit a53b9b11b3b530b884111dd770db396448577866
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Tue Aug 25 16:53:11 2020 -0800
Deployed 6e1bd6370 with MkDocs version: 1.0.4
---
404.html | 12 ++
api/index.html | 12 ++
community/index.html | 12 ++
configuration/index.html | 14 +-
custom-catalog/index.html | 12 ++
evolution/index.html | 12 ++
getting-started/index.html | 12 ++
how-to-release/index.html | 12 ++
index.html | 14 +-
java-api-quickstart/index.html | 12 ++
maintenance/index.html | 26 +++-
partitioning/index.html | 12 ++
performance/index.html | 12 ++
presto/index.html | 16 ++-
python-api-intro/index.html | 12 ++
python-feature-support/index.html | 12 ++
python-quickstart/index.html | 12 ++
releases/index.html | 12 ++
reliability/index.html | 12 ++
schemas/index.html | 12 ++
sitemap.xml | 10 ++
sitemap.xml.gz | Bin 228 -> 229 bytes
snapshots/index.html | 12 ++
.../index.html | 154 +++++++++++++--------
spark/index.html | 14 +-
spec/index.html | 12 ++
terms/index.html | 12 ++
trademarks/index.html | 12 ++
why-iceberg/index.html | 12 ++
29 files changed, 438 insertions(+), 62 deletions(-)
diff --git a/404.html b/404.html
index 63b751f..1719578 100644
--- a/404.html
+++ b/404.html
@@ -128,12 +128,24 @@
<li >
+ <a href="/spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="/presto/">Presto</a>
</li>
<li >
+ <a href="/maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="/configuration/">Configuration</a>
</li>
diff --git a/api/index.html b/api/index.html
index 96c1834..396644a 100644
--- a/api/index.html
+++ b/api/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/community/index.html b/community/index.html
index e4858e5..f275aa8 100644
--- a/community/index.html
+++ b/community/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/configuration/index.html b/configuration/index.html
index c8df522..7ecf48c 100644
--- a/configuration/index.html
+++ b/configuration/index.html
@@ -128,11 +128,23 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
+<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
<li class="active">
<a href="./">Configuration</a>
</li>
@@ -320,7 +332,7 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../presto/">
+ <a rel="prev" href="../maintenance/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
diff --git a/custom-catalog/index.html b/custom-catalog/index.html
index 9d5c9a4..8ef2bf5 100644
--- a/custom-catalog/index.html
+++ b/custom-catalog/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/evolution/index.html b/evolution/index.html
index 0548fea..c62de7e 100644
--- a/evolution/index.html
+++ b/evolution/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/getting-started/index.html b/getting-started/index.html
index 5689310..dd98e3f 100644
--- a/getting-started/index.html
+++ b/getting-started/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/how-to-release/index.html b/how-to-release/index.html
index 70aa884..2327e46 100644
--- a/how-to-release/index.html
+++ b/how-to-release/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/index.html b/index.html
index 9205b94..531b24d 100644
--- a/index.html
+++ b/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="presto/">Presto</a>
</li>
<li >
+ <a href="maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="configuration/">Configuration</a>
</li>
@@ -460,5 +472,5 @@
<!--
MkDocs version : 1.0.4
-Build Date UTC : 2020-08-25 23:07:55
+Build Date UTC : 2020-08-26 00:53:11
-->
diff --git a/java-api-quickstart/index.html b/java-api-quickstart/index.html
index 5868655..17fb4cc 100644
--- a/java-api-quickstart/index.html
+++ b/java-api-quickstart/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/maintenance/index.html b/maintenance/index.html
index 4e1fc33..1047caa 100644
--- a/maintenance/index.html
+++ b/maintenance/index.html
@@ -110,7 +110,7 @@
- <li class="dropdown">
+ <li class="dropdown active">
<a href="#" class="dropdown-toggle" data-toggle="dropdown">User docs <b class="caret"></b></a>
<ul class="dropdown-menu">
@@ -128,11 +128,23 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
+<li class="active">
+ <a href="./">Maintenance</a>
+</li>
+
+
+
<li >
<a href="../configuration/">Configuration</a>
</li>
@@ -319,6 +331,16 @@
</ul>
<ul class="nav navbar-nav navbar-right">
+ <li >
+ <a rel="prev" href="../presto/">
+ <i class="fas fa-arrow-left"></i> Previous
+ </a>
+ </li>
+ <li >
+ <a rel="next" href="../configuration/">
+ Next <i class="fas fa-arrow-right"></i>
+ </a>
+ </li>
</ul>
</div>
</div>
@@ -447,7 +469,7 @@ Actions.forTable(table).rewriteDataFiles()
<p>This example rewrites small manifests and groups data files by the first partition field.</p>
<pre><code class="java">Table table = ...
table.rewriteManifests()
- .rewriteIf(file -> file.length() < 4 * 1024 * 1024) // 10 MB
+ .rewriteIf(file -> file.length() < 10 * 1024 * 1024) // 10 MB
.clusterBy(file -> file.partition().get(0, Integer.class))
.commit();
</code></pre>
diff --git a/partitioning/index.html b/partitioning/index.html
index 15b62ea..aa0de9b 100644
--- a/partitioning/index.html
+++ b/partitioning/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/performance/index.html b/performance/index.html
index a68ca3f..0ac2e31 100644
--- a/performance/index.html
+++ b/performance/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/presto/index.html b/presto/index.html
index b8d1060..e99c4ed 100644
--- a/presto/index.html
+++ b/presto/index.html
@@ -127,6 +127,12 @@
+<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
<li class="active">
<a href="./">Presto</a>
</li>
@@ -134,6 +140,12 @@
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
@@ -320,12 +332,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../spark/">
+ <a rel="prev" href="../spark-structured-streaming/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../configuration/">
+ <a rel="next" href="../maintenance/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
diff --git a/python-api-intro/index.html b/python-api-intro/index.html
index 38136b1..ea642bd 100644
--- a/python-api-intro/index.html
+++ b/python-api-intro/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/python-feature-support/index.html b/python-feature-support/index.html
index 0535585..7cd9711 100644
--- a/python-feature-support/index.html
+++ b/python-feature-support/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/python-quickstart/index.html b/python-quickstart/index.html
index 54b6708..11c05c7 100644
--- a/python-quickstart/index.html
+++ b/python-quickstart/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/releases/index.html b/releases/index.html
index 50a7098..b781e2e 100644
--- a/releases/index.html
+++ b/releases/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/reliability/index.html b/reliability/index.html
index fcca411..8d94024 100644
--- a/reliability/index.html
+++ b/reliability/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/schemas/index.html b/schemas/index.html
index 8641cec..5052102 100644
--- a/schemas/index.html
+++ b/schemas/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/sitemap.xml b/sitemap.xml
index ad4950c..431e2c0 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -71,6 +71,16 @@
<changefreq>daily</changefreq>
</url>
<url>
+ <loc>None</loc>
+ <lastmod>2020-08-25</lastmod>
+ <changefreq>daily</changefreq>
+ </url>
+ <url>
+ <loc>None</loc>
+ <lastmod>2020-08-25</lastmod>
+ <changefreq>daily</changefreq>
+ </url>
+ <url>
<loc></loc>
<changefreq>daily</changefreq>
diff --git a/sitemap.xml.gz b/sitemap.xml.gz
index 259155f..11656c9 100644
Binary files a/sitemap.xml.gz and b/sitemap.xml.gz differ
diff --git a/snapshots/index.html b/snapshots/index.html
index 89fda98..f3aa584 100644
--- a/snapshots/index.html
+++ b/snapshots/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/partitioning/index.html b/spark-structured-streaming/index.html
similarity index 64%
copy from partitioning/index.html
copy to spark-structured-streaming/index.html
index 15b62ea..26e45c5 100644
--- a/partitioning/index.html
+++ b/spark-structured-streaming/index.html
@@ -11,7 +11,7 @@
<link rel="../img/favicon.ico">
- <title>Partitioning - Apache Iceberg</title>
+ <title>Spark Streaming - Apache Iceberg</title>
<link rel="stylesheet" href="//use.fontawesome.com/releases/v5.5.0/css/all.css" integrity="sha384-B4dIYHKNBt8Bc12p+WXckhzcICo0wtJAoU8YZTY5qE0Id1GSseTk6S+L3BlXeVIU" crossorigin="anonymous">
@@ -127,6 +127,12 @@
+<li class="active">
+ <a href="./">Spark Streaming</a>
+</li>
+
+
+
<li >
<a href="../presto/">Presto</a>
</li>
@@ -134,6 +140,12 @@
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
@@ -145,8 +157,8 @@
-<li class="active">
- <a href="./">Partitioning</a>
+<li >
+ <a href="../partitioning/">Partitioning</a>
</li>
@@ -320,12 +332,12 @@
<ul class="nav navbar-nav navbar-right">
<li >
- <a rel="prev" href="../schemas/">
+ <a rel="prev" href="../spark/">
<i class="fas fa-arrow-left"></i> Previous
</a>
</li>
<li >
- <a rel="next" href="../evolution/">
+ <a rel="next" href="../presto/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
@@ -339,14 +351,19 @@
<div class="col-md-3"><div class="bs-sidebar hidden-print affix well" role="complementary">
<ul class="nav bs-sidenav">
- <li class="first-level active"><a href="#what-is-partitioning">What is partitioning?</a></li>
- <li class="first-level "><a href="#what-does-iceberg-do-differently">What does Iceberg do differently?</a></li>
- <li class="second-level"><a href="#partitioning-in-hive">Partitioning in Hive</a></li>
+ <li class="first-level active"><a href="#spark-structured-streaming">Spark Structured Streaming</a></li>
+ <li class="second-level"><a href="#streaming-writes">Streaming Writes</a></li>
- <li class="second-level"><a href="#problems-with-hive-partitioning">Problems with Hive partitioning</a></li>
+ <li class="second-level"><a href="#maintenance-for-streaming-tables">Maintenance for streaming tables</a></li>
- <li class="second-level"><a href="#icebergs-hidden-partitioning">Iceberg's hidden partitioning</a></li>
+ <li class="third-level"><a href="#tune-the-rate-of-commits">Tune the rate of commits</a></li>
+ <li class="third-level"><a href="#expire-old-snapshots">Expire old snapshots</a></li>
+ <li class="third-level"><a href="#compacting-data-files">Compacting data files</a></li>
+ <li class="third-level"><a href="#rewrite-manifests">Rewrite manifests</a></li>
+ <li class="second-level"><a href="#appendix">Appendix</a></li>
+ <li class="third-level"><a href="#retrieve-table-instance-in-hadoop-catalog">Retrieve Table instance in Hadoop catalog</a></li>
+ <li class="third-level"><a href="#retrieve-table-instance-in-hive-catalog">Retrieve Table instance in Hive catalog</a></li>
</ul>
</div></div>
<div class="col-md-9" role="main">
@@ -368,58 +385,85 @@
- limitations under the License.
-->
-<h2 id="what-is-partitioning">What is partitioning?<a class="headerlink" href="#what-is-partitioning" title="Permanent link">¶</a></h2>
-<p>Partitioning is a way to make queries faster by grouping similar rows together when writing.</p>
-<p>For example, queries for log entries from a <code>logs</code> table would usually include a time range, like this query for logs between 10 and 12 AM:</p>
-<pre><code class="sql">SELECT level, message FROM logs
-WHERE event_time BETWEEN '2018-12-01 10:00:00' AND '2018-12-01 12:00:00'
+<h1 id="spark-structured-streaming">Spark Structured Streaming<a class="headerlink" href="#spark-structured-streaming" title="Permanent link">¶</a></h1>
+<p>Iceberg uses Apache Spark’s DataSourceV2 API for data source and catalog implementations. Spark DSv2 is an evolving API
+with different levels of support in Spark versions.</p>
+<p>As of Spark 3.0, DataFrame reads and writes are supported.</p>
+<table>
+<thead>
+<tr>
+<th>Feature support</th>
+<th>Spark 3.0</th>
+<th>Spark 2.4</th>
+<th>Notes</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><a href="#writing-with-streaming-query">DataFrame write</a></td>
+<td>✔</td>
+<td>✔</td>
+<td></td>
+</tr>
+</tbody>
+</table>
+<h2 id="streaming-writes">Streaming Writes<a class="headerlink" href="#streaming-writes" title="Permanent link">¶</a></h2>
+<p>To write values from streaming query to Iceberg table, use <code>DataStreamWriter</code>:</p>
+<pre><code class="scala">val tableIdentifier: String = ...
+data.writeStream
+ .format("iceberg")
+ .outputMode("append")
+ .trigger(Trigger.ProcessingTime(1, TimeUnit.MINUTES))
+ .option("path", tableIdentifier)
+ .option("checkpointLocation", checkpointPath)
+ .start()
</code></pre>
-<p>Configuring the <code>logs</code> table to partition by the date of <code>event_time</code> will group log events into files with the same event date. Iceberg keeps track of that date and will use it to skip files for other dates that don’t have useful data.</p>
-<p>Iceberg can partition timestamps by year, month, day, and hour granularity. It can also use a categorical column, like <code>level</code> in this logs example, to store rows together and speed up queries.</p>
-<h2 id="what-does-iceberg-do-differently">What does Iceberg do differently?<a class="headerlink" href="#what-does-iceberg-do-differently" title="Permanent link">¶</a></h2>
-<p>Other tables formats like Hive support partitioning, but Iceberg supports <em>hidden partitioning</em>.</p>
+<p>The <code>tableIdentifier</code> can be:</p>
+<ul>
+<li>The fully-qualified path to a HDFS table, like <code>hdfs://nn:8020/path/to/table</code></li>
+<li>A table name if the table is tracked by a catalog, like <code>database.table_name</code></li>
+</ul>
+<p>Iceberg doesn’t support “continuous processing”, as it doesn’t provide the interface to “commit” the output.</p>
+<p>Iceberg supports <code>append</code> and <code>complete</code> output modes:</p>
<ul>
-<li>Iceberg handles the tedious and error-prone task of producing partition values for rows in a table.</li>
-<li>Iceberg avoids reading unnecessary partitions automatically. Consumers don’t need to know how the table is partitioned and add extra filters to their queries.</li>
-<li>Iceberg partition layouts can evolve as needed.</li>
+<li><code>append</code>: appends the rows of every micro-batch to the table</li>
+<li><code>complete</code>: replaces the table contents every micro-batch</li>
</ul>
-<h3 id="partitioning-in-hive">Partitioning in Hive<a class="headerlink" href="#partitioning-in-hive" title="Permanent link">¶</a></h3>
-<p>To demonstrate the difference, consider how Hive would handle a <code>logs</code> table.</p>
-<p>In Hive, partitions are explicit and appear as a column, so the <code>logs</code> table would have a column called <code>event_date</code>. When writing, an insert needs to supply the data for the <code>event_date</code> column:</p>
-<pre><code class="sql">INSERT INTO logs PARTITION (event_date)
- SELECT level, message, event_time, format_time(event_time, 'YYYY-MM-dd')
- FROM unstructured_log_source
+<p>The table should be created in prior to start the streaming query. Refer <a href="/spark/#create-table">SQL create table</a>
+on Spark page to see how to create the Iceberg table.</p>
+<h2 id="maintenance-for-streaming-tables">Maintenance for streaming tables<a class="headerlink" href="#maintenance-for-streaming-tables" title="Permanent link">¶</a></h2>
+<p>Streaming queries can create new table versions quickly, which creates lots of table metadata to track those versions.
+Maintaining metadata by tuning the rate of commits, expiring old snapshots, and automatically cleaning up metadata files
+is highly recommended.</p>
+<h3 id="tune-the-rate-of-commits">Tune the rate of commits<a class="headerlink" href="#tune-the-rate-of-commits" title="Permanent link">¶</a></h3>
+<p>Having high rate of commits would produce lots of data files, manifests, and snapshots which leads the table hard
+to maintain. We encourage having trigger interval 1 minute at minimum, and increase the interval if needed.</p>
+<p>The triggers section in <a href="https://spark.apache.org/docs/latest/structured-streaming-programming-guide.html#triggers">Structured Streaming Programming Guide</a>
+documents how to configure the interval.</p>
+<h3 id="expire-old-snapshots">Expire old snapshots<a class="headerlink" href="#expire-old-snapshots" title="Permanent link">¶</a></h3>
+<p>Each micro-batch written to a table produces a new snapshot, which are tracked in table metadata until they are expired to remove the metadata and any data files that are no longer needed. Snapshots accumulate quickly with frequent commits, so it is highly recommended that tables written by streaming queries are <a href="../maintenance#expire-snapshots">regularly maintained</a>.</p>
+<h3 id="compacting-data-files">Compacting data files<a class="headerlink" href="#compacting-data-files" title="Permanent link">¶</a></h3>
+<p>The amount of data written in a micro batch is typically small, which can cause the table metadata to track lots of small files. Compacting small files into larger files reduces the metadata needed by the table, and increases query efficiency.</p>
+<h3 id="rewrite-manifests">Rewrite manifests<a class="headerlink" href="#rewrite-manifests" title="Permanent link">¶</a></h3>
+<p>To optimize write latency on streaming workload, Iceberg may write the new snapshot with a “fast” append that does not automatically compact manifests.
+This could lead lots of small manifest files. Manifests can be <a href="../maintenance#rewrite-manifests">rewritten to optimize queries and to compact</a>.</p>
+<h2 id="appendix">Appendix<a class="headerlink" href="#appendix" title="Permanent link">¶</a></h2>
+<h3 id="retrieve-table-instance-in-hadoop-catalog">Retrieve Table instance in Hadoop catalog<a class="headerlink" href="#retrieve-table-instance-in-hadoop-catalog" title="Permanent link">¶</a></h3>
+<pre><code class="scala">// ... assume catalogName, dbName, tableName are present ...
+import org.apache.iceberg.hadoop.HadoopTables
+val warehousePath = spark.sparkContext.getConf.get(s"spark.sql.catalog.$catalogName.warehouse")
+val hadoopTables = new HadoopTables(spark.sessionState.newHadoopConf)
+val table = hadoopTables.load(s"$warehousePath/$dbName/$tableName")
</code></pre>
-<p>Similarly, queries that search through the <code>logs</code> table must have an <code>event_date</code> filter in addition to an <code>event_time</code> filter.</p>
-<pre><code class="sql">SELECT level, count(1) as count FROM logs
-WHERE event_time BETWEEN '2018-12-01 10:00:00' AND '2018-12-01 12:00:00'
- AND event_date = '2018-12-01'
-</code></pre>
+<h3 id="retrieve-table-instance-in-hive-catalog">Retrieve Table instance in Hive catalog<a class="headerlink" href="#retrieve-table-instance-in-hive-catalog" title="Permanent link">¶</a></h3>
+<pre><code class="scala">// ... assume dbName, tableName are present ...
+import org.apache.iceberg.hive.HiveCatalog;
-<p>If the <code>event_date</code> filter were missing, Hive would scan through every file in the table because it doesn’t know that the <code>event_time</code> column is related to the <code>event_date</code> column.</p>
-<h3 id="problems-with-hive-partitioning">Problems with Hive partitioning<a class="headerlink" href="#problems-with-hive-partitioning" title="Permanent link">¶</a></h3>
-<p>Hive must be given partition values. In the logs example, it doesn’t know the relationship between <code>event_time</code> and <code>event_date</code>.</p>
-<p>This leads to several problems:</p>
-<ul>
-<li>Hive can’t validate partition values – it is up to the writer to produce the correct value<ul>
-<li>Using the wrong format, <code>2018-12-01</code> instead of <code>20181201</code>, produces silently incorrect results, not query failures</li>
-<li>Using the wrong source column, like <code>processing_time</code>, or time zone also causes incorrect results, not failures</li>
-</ul>
-</li>
-<li>It is up to the user to write queries correctly<ul>
-<li>Using the wrong format also leads to silently incorrect results</li>
-<li>Users that don’t understand a table’s physical layout get needlessly slow queries – Hive can’t translate filters automatically</li>
-</ul>
-</li>
-<li>Working queries are tied to the table’s partitioning scheme, so partitioning configuration cannot be changed without breaking queries</li>
-</ul>
-<h3 id="icebergs-hidden-partitioning">Iceberg’s hidden partitioning<a class="headerlink" href="#icebergs-hidden-partitioning" title="Permanent link">¶</a></h3>
-<p>Iceberg produces partition values by taking a column value and optionally transforming it. Iceberg is responsible for converting <code>event_time</code> into <code>event_date</code>, and keeps track of the relationship.</p>
-<p>Table partitioning is configured using these relationships. The <code>logs</code> table would be partitioned by <code>date(event_time)</code> and <code>level</code>.</p>
-<p>Because Iceberg doesn’t require user-maintained partition columns, it can hide partitioning. Partition values are produced correctly every time and always used to speed up queries, when possible. Producers and consumers wouldn’t even see <code>event_date</code>.</p>
-<p>Most importantly, queries no longer depend on a table’s physical layout. With a separation between physical and logical, Iceberg tables can evolve partition schemes over time as data volume changes. Misconfigured tables can be fixed without an expensive migration.</p></div>
+val catalog = new HiveCatalog(spark.sessionState.newHadoopConf)
+val table = catalog.loadTable(TableIdentifier.of(dbName, tableName))
+</code></pre></div>
</div>
diff --git a/spark/index.html b/spark/index.html
index 25802f8..aa2d355 100644
--- a/spark/index.html
+++ b/spark/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
@@ -325,7 +337,7 @@
</a>
</li>
<li >
- <a rel="next" href="../presto/">
+ <a rel="next" href="../spark-structured-streaming/">
Next <i class="fas fa-arrow-right"></i>
</a>
</li>
diff --git a/spec/index.html b/spec/index.html
index c647538..5170a2c 100644
--- a/spec/index.html
+++ b/spec/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/terms/index.html b/terms/index.html
index 7688eb7..adb8a54 100644
--- a/terms/index.html
+++ b/terms/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/trademarks/index.html b/trademarks/index.html
index ea15ac5..695484d 100644
--- a/trademarks/index.html
+++ b/trademarks/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>
diff --git a/why-iceberg/index.html b/why-iceberg/index.html
index 0a958d8..02f3468 100644
--- a/why-iceberg/index.html
+++ b/why-iceberg/index.html
@@ -128,12 +128,24 @@
<li >
+ <a href="../spark-structured-streaming/">Spark Streaming</a>
+</li>
+
+
+
+<li >
<a href="../presto/">Presto</a>
</li>
<li >
+ <a href="../maintenance/">Maintenance</a>
+</li>
+
+
+
+<li >
<a href="../configuration/">Configuration</a>
</li>