You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by br...@apache.org on 2020/02/04 19:41:46 UTC
[drill] 03/03: More changes after code review
This is an automated email from the ASF dual-hosted git repository.
bridgetb pushed a commit to branch gh-pages
in repository https://gitbox.apache.org/repos/asf/drill.git
commit fd5041dab391c4d5c66d3bf6b442fb8f901d4217
Author: Volodymyr Vysotskyi <vv...@gmail.com>
AuthorDate: Fri Jan 31 18:33:43 2020 +0200
More changes after code review
---
_data/docs.json | 308 ++++++++++-----------
.../drill-metastore/010-using-drill-metastore.md | 100 ++++---
.../drill-metastore/030-drill-iceberg-metastore.md | 14 +-
3 files changed, 229 insertions(+), 193 deletions(-)
diff --git a/_data/docs.json b/_data/docs.json
index b62691b..0b74254 100644
--- a/_data/docs.json
+++ b/_data/docs.json
@@ -3171,8 +3171,8 @@
}
],
"children": [],
- "next_title": "Drill Metastore",
- "next_url": "/docs/drill-metastore/",
+ "next_title": "Identifying Performance Issues",
+ "next_url": "/docs/identifying-performance-issues/",
"parent": "Query Plans and Tuning",
"previous_title": "Throttling",
"previous_url": "/docs/throttling/",
@@ -4572,12 +4572,12 @@
}
],
"children": [],
- "next_title": "Identifying Performance Issues",
- "next_url": "/docs/identifying-performance-issues/",
+ "next_title": "Performance Tuning Introduction",
+ "next_url": "/docs/performance-tuning-introduction/",
"parent": "Drill Metastore",
"previous_title": "Using Drill Metastore",
"previous_url": "/docs/using-drill-metastore/",
- "relative_path": "_docs/performance-tuning/drill-metastore/020-drill-iceberg-metastore.md",
+ "relative_path": "_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md",
"title": "Drill Iceberg Metastore",
"url": "/docs/drill-iceberg-metastore/"
},
@@ -4639,12 +4639,12 @@
}
],
"children": [],
- "next_title": "Identifying Performance Issues",
- "next_url": "/docs/identifying-performance-issues/",
+ "next_title": "Performance Tuning Introduction",
+ "next_url": "/docs/performance-tuning-introduction/",
"parent": "Drill Metastore",
"previous_title": "Using Drill Metastore",
"previous_url": "/docs/using-drill-metastore/",
- "relative_path": "_docs/performance-tuning/drill-metastore/020-drill-iceberg-metastore.md",
+ "relative_path": "_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md",
"title": "Drill Iceberg Metastore",
"url": "/docs/drill-iceberg-metastore/"
}
@@ -4652,9 +4652,9 @@
"next_title": "Using Drill Metastore",
"next_url": "/docs/using-drill-metastore/",
"parent": "Performance Tuning",
- "previous_title": "Controlling Parallelization to Balance Performance with Multi-Tenancy",
- "previous_url": "/docs/controlling-parallelization-to-balance-performance-with-multi-tenancy/",
- "relative_path": "_docs/performance-tuning/050-drill-metastore.md",
+ "previous_title": "Performance Tuning",
+ "previous_url": "/docs/performance-tuning/",
+ "relative_path": "_docs/performance-tuning/005-drill-metastore.md",
"title": "Drill Metastore",
"url": "/docs/drill-metastore/"
},
@@ -5559,8 +5559,8 @@
"next_title": "Query Plans",
"next_url": "/docs/query-plans/",
"parent": "Performance Tuning",
- "previous_title": "Drill Iceberg Metastore",
- "previous_url": "/docs/drill-iceberg-metastore/",
+ "previous_title": "Controlling Parallelization to Balance Performance with Multi-Tenancy",
+ "previous_url": "/docs/controlling-parallelization-to-balance-performance-with-multi-tenancy/",
"relative_path": "_docs/performance-tuning/050-identifying-performance-issues.md",
"title": "Identifying Performance Issues",
"url": "/docs/identifying-performance-issues/"
@@ -8185,12 +8185,72 @@
"url": "/docs/performance-tuning/"
}
],
+ "children": [
+ {
+ "breadcrumbs": [
+ {
+ "title": "Drill Metastore",
+ "url": "/docs/drill-metastore/"
+ },
+ {
+ "title": "Performance Tuning",
+ "url": "/docs/performance-tuning/"
+ }
+ ],
+ "children": [],
+ "next_title": "Drill Iceberg Metastore",
+ "next_url": "/docs/drill-iceberg-metastore/",
+ "parent": "Drill Metastore",
+ "previous_title": "Drill Metastore",
+ "previous_url": "/docs/drill-metastore/",
+ "relative_path": "_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md",
+ "title": "Using Drill Metastore",
+ "url": "/docs/using-drill-metastore/"
+ },
+ {
+ "breadcrumbs": [
+ {
+ "title": "Drill Metastore",
+ "url": "/docs/drill-metastore/"
+ },
+ {
+ "title": "Performance Tuning",
+ "url": "/docs/performance-tuning/"
+ }
+ ],
+ "children": [],
+ "next_title": "Performance Tuning Introduction",
+ "next_url": "/docs/performance-tuning-introduction/",
+ "parent": "Drill Metastore",
+ "previous_title": "Using Drill Metastore",
+ "previous_url": "/docs/using-drill-metastore/",
+ "relative_path": "_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md",
+ "title": "Drill Iceberg Metastore",
+ "url": "/docs/drill-iceberg-metastore/"
+ }
+ ],
+ "next_title": "Using Drill Metastore",
+ "next_url": "/docs/using-drill-metastore/",
+ "parent": "Performance Tuning",
+ "previous_title": "Performance Tuning",
+ "previous_url": "/docs/performance-tuning/",
+ "relative_path": "_docs/performance-tuning/005-drill-metastore.md",
+ "title": "Drill Metastore",
+ "url": "/docs/drill-metastore/"
+ },
+ {
+ "breadcrumbs": [
+ {
+ "title": "Performance Tuning",
+ "url": "/docs/performance-tuning/"
+ }
+ ],
"children": [],
"next_title": "Partition Pruning",
"next_url": "/docs/partition-pruning/",
"parent": "Performance Tuning",
- "previous_title": "Performance Tuning",
- "previous_url": "/docs/performance-tuning/",
+ "previous_title": "Drill Iceberg Metastore",
+ "previous_url": "/docs/drill-iceberg-metastore/",
"relative_path": "_docs/performance-tuning/010-performance-tuning-introduction.md",
"title": "Performance Tuning Introduction",
"url": "/docs/performance-tuning-introduction/"
@@ -8507,8 +8567,8 @@
}
],
"children": [],
- "next_title": "Drill Metastore",
- "next_url": "/docs/drill-metastore/",
+ "next_title": "Identifying Performance Issues",
+ "next_url": "/docs/identifying-performance-issues/",
"parent": "Query Plans and Tuning",
"previous_title": "Throttling",
"previous_url": "/docs/throttling/",
@@ -8537,66 +8597,6 @@
{
"breadcrumbs": [
{
- "title": "Drill Metastore",
- "url": "/docs/drill-metastore/"
- },
- {
- "title": "Performance Tuning",
- "url": "/docs/performance-tuning/"
- }
- ],
- "children": [],
- "next_title": "Drill Iceberg Metastore",
- "next_url": "/docs/drill-iceberg-metastore/",
- "parent": "Drill Metastore",
- "previous_title": "Drill Metastore",
- "previous_url": "/docs/drill-metastore/",
- "relative_path": "_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md",
- "title": "Using Drill Metastore",
- "url": "/docs/using-drill-metastore/"
- },
- {
- "breadcrumbs": [
- {
- "title": "Drill Metastore",
- "url": "/docs/drill-metastore/"
- },
- {
- "title": "Performance Tuning",
- "url": "/docs/performance-tuning/"
- }
- ],
- "children": [],
- "next_title": "Identifying Performance Issues",
- "next_url": "/docs/identifying-performance-issues/",
- "parent": "Drill Metastore",
- "previous_title": "Using Drill Metastore",
- "previous_url": "/docs/using-drill-metastore/",
- "relative_path": "_docs/performance-tuning/drill-metastore/020-drill-iceberg-metastore.md",
- "title": "Drill Iceberg Metastore",
- "url": "/docs/drill-iceberg-metastore/"
- }
- ],
- "next_title": "Using Drill Metastore",
- "next_url": "/docs/using-drill-metastore/",
- "parent": "Performance Tuning",
- "previous_title": "Controlling Parallelization to Balance Performance with Multi-Tenancy",
- "previous_url": "/docs/controlling-parallelization-to-balance-performance-with-multi-tenancy/",
- "relative_path": "_docs/performance-tuning/050-drill-metastore.md",
- "title": "Drill Metastore",
- "url": "/docs/drill-metastore/"
- },
- {
- "breadcrumbs": [
- {
- "title": "Performance Tuning",
- "url": "/docs/performance-tuning/"
- }
- ],
- "children": [
- {
- "breadcrumbs": [
- {
"title": "Identifying Performance Issues",
"url": "/docs/identifying-performance-issues/"
},
@@ -8640,8 +8640,8 @@
"next_title": "Query Plans",
"next_url": "/docs/query-plans/",
"parent": "Performance Tuning",
- "previous_title": "Drill Iceberg Metastore",
- "previous_url": "/docs/drill-iceberg-metastore/",
+ "previous_title": "Controlling Parallelization to Balance Performance with Multi-Tenancy",
+ "previous_url": "/docs/controlling-parallelization-to-balance-performance-with-multi-tenancy/",
"relative_path": "_docs/performance-tuning/050-identifying-performance-issues.md",
"title": "Identifying Performance Issues",
"url": "/docs/identifying-performance-issues/"
@@ -8724,8 +8724,8 @@
"url": "/docs/monitoring-metrics/"
}
],
- "next_title": "Performance Tuning Introduction",
- "next_url": "/docs/performance-tuning-introduction/",
+ "next_title": "Drill Metastore",
+ "next_url": "/docs/drill-metastore/",
"parent": "",
"previous_title": "Monitoring and Canceling Queries in the Drill Web UI",
"previous_url": "/docs/monitoring-and-canceling-queries-in-the-drill-web-ui/",
@@ -8744,8 +8744,8 @@
"next_title": "Partition Pruning",
"next_url": "/docs/partition-pruning/",
"parent": "Performance Tuning",
- "previous_title": "Performance Tuning",
- "previous_url": "/docs/performance-tuning/",
+ "previous_title": "Drill Iceberg Metastore",
+ "previous_url": "/docs/drill-iceberg-metastore/",
"relative_path": "_docs/performance-tuning/010-performance-tuning-introduction.md",
"title": "Performance Tuning Introduction",
"url": "/docs/performance-tuning-introduction/"
@@ -9793,8 +9793,8 @@
}
],
"children": [],
- "next_title": "Drill Metastore",
- "next_url": "/docs/drill-metastore/",
+ "next_title": "Identifying Performance Issues",
+ "next_url": "/docs/identifying-performance-issues/",
"parent": "Query Plans and Tuning",
"previous_title": "Throttling",
"previous_url": "/docs/throttling/",
@@ -19675,12 +19675,72 @@
"url": "/docs/performance-tuning/"
}
],
+ "children": [
+ {
+ "breadcrumbs": [
+ {
+ "title": "Drill Metastore",
+ "url": "/docs/drill-metastore/"
+ },
+ {
+ "title": "Performance Tuning",
+ "url": "/docs/performance-tuning/"
+ }
+ ],
+ "children": [],
+ "next_title": "Drill Iceberg Metastore",
+ "next_url": "/docs/drill-iceberg-metastore/",
+ "parent": "Drill Metastore",
+ "previous_title": "Drill Metastore",
+ "previous_url": "/docs/drill-metastore/",
+ "relative_path": "_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md",
+ "title": "Using Drill Metastore",
+ "url": "/docs/using-drill-metastore/"
+ },
+ {
+ "breadcrumbs": [
+ {
+ "title": "Drill Metastore",
+ "url": "/docs/drill-metastore/"
+ },
+ {
+ "title": "Performance Tuning",
+ "url": "/docs/performance-tuning/"
+ }
+ ],
+ "children": [],
+ "next_title": "Performance Tuning Introduction",
+ "next_url": "/docs/performance-tuning-introduction/",
+ "parent": "Drill Metastore",
+ "previous_title": "Using Drill Metastore",
+ "previous_url": "/docs/using-drill-metastore/",
+ "relative_path": "_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md",
+ "title": "Drill Iceberg Metastore",
+ "url": "/docs/drill-iceberg-metastore/"
+ }
+ ],
+ "next_title": "Using Drill Metastore",
+ "next_url": "/docs/using-drill-metastore/",
+ "parent": "Performance Tuning",
+ "previous_title": "Performance Tuning",
+ "previous_url": "/docs/performance-tuning/",
+ "relative_path": "_docs/performance-tuning/005-drill-metastore.md",
+ "title": "Drill Metastore",
+ "url": "/docs/drill-metastore/"
+ },
+ {
+ "breadcrumbs": [
+ {
+ "title": "Performance Tuning",
+ "url": "/docs/performance-tuning/"
+ }
+ ],
"children": [],
"next_title": "Partition Pruning",
"next_url": "/docs/partition-pruning/",
"parent": "Performance Tuning",
- "previous_title": "Performance Tuning",
- "previous_url": "/docs/performance-tuning/",
+ "previous_title": "Drill Iceberg Metastore",
+ "previous_url": "/docs/drill-iceberg-metastore/",
"relative_path": "_docs/performance-tuning/010-performance-tuning-introduction.md",
"title": "Performance Tuning Introduction",
"url": "/docs/performance-tuning-introduction/"
@@ -19997,8 +20057,8 @@
}
],
"children": [],
- "next_title": "Drill Metastore",
- "next_url": "/docs/drill-metastore/",
+ "next_title": "Identifying Performance Issues",
+ "next_url": "/docs/identifying-performance-issues/",
"parent": "Query Plans and Tuning",
"previous_title": "Throttling",
"previous_url": "/docs/throttling/",
@@ -20027,66 +20087,6 @@
{
"breadcrumbs": [
{
- "title": "Drill Metastore",
- "url": "/docs/drill-metastore/"
- },
- {
- "title": "Performance Tuning",
- "url": "/docs/performance-tuning/"
- }
- ],
- "children": [],
- "next_title": "Drill Iceberg Metastore",
- "next_url": "/docs/drill-iceberg-metastore/",
- "parent": "Drill Metastore",
- "previous_title": "Drill Metastore",
- "previous_url": "/docs/drill-metastore/",
- "relative_path": "_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md",
- "title": "Using Drill Metastore",
- "url": "/docs/using-drill-metastore/"
- },
- {
- "breadcrumbs": [
- {
- "title": "Drill Metastore",
- "url": "/docs/drill-metastore/"
- },
- {
- "title": "Performance Tuning",
- "url": "/docs/performance-tuning/"
- }
- ],
- "children": [],
- "next_title": "Identifying Performance Issues",
- "next_url": "/docs/identifying-performance-issues/",
- "parent": "Drill Metastore",
- "previous_title": "Using Drill Metastore",
- "previous_url": "/docs/using-drill-metastore/",
- "relative_path": "_docs/performance-tuning/drill-metastore/020-drill-iceberg-metastore.md",
- "title": "Drill Iceberg Metastore",
- "url": "/docs/drill-iceberg-metastore/"
- }
- ],
- "next_title": "Using Drill Metastore",
- "next_url": "/docs/using-drill-metastore/",
- "parent": "Performance Tuning",
- "previous_title": "Controlling Parallelization to Balance Performance with Multi-Tenancy",
- "previous_url": "/docs/controlling-parallelization-to-balance-performance-with-multi-tenancy/",
- "relative_path": "_docs/performance-tuning/050-drill-metastore.md",
- "title": "Drill Metastore",
- "url": "/docs/drill-metastore/"
- },
- {
- "breadcrumbs": [
- {
- "title": "Performance Tuning",
- "url": "/docs/performance-tuning/"
- }
- ],
- "children": [
- {
- "breadcrumbs": [
- {
"title": "Identifying Performance Issues",
"url": "/docs/identifying-performance-issues/"
},
@@ -20130,8 +20130,8 @@
"next_title": "Query Plans",
"next_url": "/docs/query-plans/",
"parent": "Performance Tuning",
- "previous_title": "Drill Iceberg Metastore",
- "previous_url": "/docs/drill-iceberg-metastore/",
+ "previous_title": "Controlling Parallelization to Balance Performance with Multi-Tenancy",
+ "previous_url": "/docs/controlling-parallelization-to-balance-performance-with-multi-tenancy/",
"relative_path": "_docs/performance-tuning/050-identifying-performance-issues.md",
"title": "Identifying Performance Issues",
"url": "/docs/identifying-performance-issues/"
@@ -20214,8 +20214,8 @@
"url": "/docs/monitoring-metrics/"
}
],
- "next_title": "Performance Tuning Introduction",
- "next_url": "/docs/performance-tuning-introduction/",
+ "next_title": "Drill Metastore",
+ "next_url": "/docs/drill-metastore/",
"parent": "",
"previous_title": "Monitoring and Canceling Queries in the Drill Web UI",
"previous_url": "/docs/monitoring-and-canceling-queries-in-the-drill-web-ui/",
@@ -23026,4 +23026,4 @@
"url": "/docs/ecosystem/"
}
]
-}
+}
\ No newline at end of file
diff --git a/_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md b/_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md
index d1dceb7..42f93c1 100644
--- a/_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md
+++ b/_docs/performance-tuning/drill-metastore/010-using-drill-metastore.md
@@ -1,12 +1,12 @@
---
title: "Using Drill Metastore"
parent: "Drill Metastore"
-date: 2020-01-30
+date: 2020-01-31
---
Drill 1.17 introduces the Drill Metastore which stores the table schema and table statistics. Statistics allow Drill to better create optimal query plans.
-The Metastore is an Beta feature; it is subject to change. We encourage you to try it and provide feedback.
+The Metastore is a Beta feature; it is subject to change. We encourage you to try it and provide feedback.
Because the Metastore is in Beta, the SQL commands and Metastore formats may change in the next release.
{% include startnote.html %}In Drill 1.17, this feature is supported for Parquet tables only and is disabled by default.{% include endnote.html %}
@@ -27,12 +27,10 @@ Once you enable the Metastore, the next step is to populate it with data. Drill
query performance. In general, large tables benefit from statistics more than small tables do.
Unlike Hive, Drill does not require you to declare a schema. Instead, Drill infers the schema by scanning your table
- and computes some metadata like MIN / MAX column values and NULLS COUNT designated as "metadata" to be able to
- produce more optimizations like filter push-down, etc. If `planner.statistics.use` option is enabled, this command
- will also calculate and store table statistics into Drill Metastore.
-
-Unlike Hive, Drill does not require you to declare a schema. Instead, Drill infers the schema by scanning your table
- in the same way as it is done during regular select.
+ in the same way as it is done during regular select and computes some metadata like `MIN` / `MAX` column values and
+ `NULLS_COUNT` designated as "metadata" to be able to produce more optimizations like filter push-down, etc. If
+ `planner.statistics.use` option is enabled, this command will also calculate and store table statistics into Drill
+ Metastore.
## Configuration
@@ -42,13 +40,24 @@ indicated in `drill-metastore-distrib.conf`.
All configuration properties should reside in `drill.metastore` namespace.
Metastore implementation based on class implementation config property `drill.metastore.implementation.class`.
+The default value is the following:
+
+```
+drill.metastore: {
+ implementation.class: "org.apache.drill.metastore.iceberg.IcebergMetastore"
+}
+```
+
+Note, that currently out of box Iceberg Metastore is available and is the default one. Though any custom
+ implementation can be added by placing the JAR into classpath which has the implementation of
+ `org.apache.drill.metastore.Metastore` interface and indicating custom class in the `drill.metastore.implementation.class`.
### Metastore Components
-Metastore can store metadata for various components: tables, views etc.
+Metastore can store metadata for various components: tables, views, etc.
Current implementation provides fully functioning support for tables component.
Views component support is not implemented but contains stub methods to show
-how new Metastore components like udfs, storage plugins, etc. be added in future.
+how new Metastore components like UDFs, storage plugins, etc. can be added in the future.
### Metastore Tables
@@ -56,15 +65,15 @@ Metastore Tables component contains metadata about Drill tables, including gener
information about table segments, files, row groups, partitions.
Full table metadata consists of two major concepts: general information and top-level segments metadata.
-Table general information contains basic table information and corresponds to `BaseTableMetadata` class.
+Table general information contains basic table information and corresponds to the `BaseTableMetadata` class.
-Table can be non-partitioned and partitioned. Non-partitioned tables, have only one top-level segment
+A table can be non-partitioned and partitioned. Non-partitioned tables have only one top-level segment
which is called default (`MetadataInfo#DEFAULT_SEGMENT_KEY`). Partitioned tables may have several top-level segments.
-Each top-level segment can include metadata about inner segments, files, row groups and partitions.
+Each top-level segment can include metadata about inner segments, files, row groups, and partitions.
-Unique table identifier in Metastore Tables is combination of storage plugin, workspace and table name.
+A unique table identifier in Metastore Tables is a combination of storage plugin, workspace, and table name.
Table metadata inside is grouped by top-level segments, unique identifier of the top-level segment and its metadata
-is storage plugin, workspace, table name and metadata key.
+is storage plugin, workspace, table name, and metadata key.
### Related Session/System Options
@@ -74,7 +83,7 @@ The following options are set via `ALTER SYSTEM SET`, or `ALTER SESSION SET` or
Enables Drill Metastore usage to be able to store table metadata during ANALYZE TABLE commands execution and to be able
to read table metadata during regular queries execution or when querying some INFORMATION_SCHEMA tables. Default is `false`.
- **metastore.metadata.store.depth_level**
-Specifies maximum level depth for collecting metadata. Same options as the _level_ option above. Default is `'ALL'`.
+Specifies the maximum level of depth for collecting metadata. Same options as the _level_ option above. Default is `'ALL'`.
- **metastore.retrieval.retry_attempts**
If you run the `ANALYZE TABLE` command at the same time as queries run, then the query can read incorrect or corrupt statistics.
Drill will reload statistics and replan the query. This option specifies the maximum number of retry attempts. Default is `5`.
@@ -91,19 +100,32 @@ Enable `planner.statistics.use` to be able to use statistics during query planni
- **metastore.metadata.ctas.auto-collect**
Drill provides the [`CREATE TABLE AS`]({{site.baseurl}}/docs/create-or-replace-schema) commands to create new tables.
This option causes Drill to gather schema and statistics for those tables automatically as they are written.
-This option is not active for now. Possible values: `'ALL'`, `'SCHEMA'`, `'NONE'`. Default is `'NONE'`.
+This option is not implemented for now. Possible values: `'ALL'`, `'SCHEMA'`, `'NONE'`. Default is `'NONE'`.
- **drill.exec.storage.implicit.last_modified_time.column.label**
-Sets the implicit column name for the last modified time (`lmt`) column. Used when producing Metastore analyze.
+Sets the implicit column name for the last modified time (`lmt`) column. Used when producing Metastore analyze. You can
+ set the last modified time column name to custom name when current column name clashes which column name present in the
+ table. If your table contains a column name with the same name as an implicit column, the implicit column takes
+ priority and shadows column from the table.
- **drill.exec.storage.implicit.row_group_index.column.label**
-Sets the implicit column name for the row group index (`rgi`) column. Used when producing Metastore analyze.
+Sets the implicit column name for the row group index (`rgi`) column. Used when producing Metastore analyze. You can
+ set row group index column name to custom name when current column name clashes which column name present in the
+ table. If your table contains a column name with the same name as an implicit column, the implicit column takes
+ priority and shadows column from the table.
- **drill.exec.storage.implicit.row_group_length.column.label**
-Sets the implicit column name for the row group length (`rgl`) column. Used when producing Metastore analyze.
+Sets the implicit column name for the row group length (`rgl`) column. Used when producing Metastore analyze. You can
+ set row group length column name to custom name when current column name clashes which column name present in the
+ table. If your table contains a column name with the same name as an implicit column, the implicit column takes
+ priority and shadows column from the table.
- **drill.exec.storage.implicit.row_group_start.column.label**
-Sets the implicit column name for the row group start (`rgs`) column. Used when producing Metastore analyze.
+Sets the implicit column name for the row group start (`rgs`) column. Used when producing Metastore analyze. You can
+ set row group start column name to custom name when current column name clashes which column name present in the
+ table. If your table contains a column name with the same name as an implicit column, the implicit column takes
+ priority and shadows column from the table.
## Incremental analysis
-If you have computed statistics for a table, and issue `ANALYZE TABLE` a second time, Drill will attempt to update statistics, called "incremental analysis."
+If you have computed statistics for a table, and issue `ANALYZE TABLE` a second time, Drill will attempt to update
+ statistics, called "incremental analysis."
Incremental analysis will compute metadata only for files and partitions changed since the last analysis and reuse
actual metadata from the Metastore where possible.
@@ -116,9 +138,9 @@ If either of these two conditions is false, Drill will perform a full analysis o
## General Information
- Drill 1.17 supports the Metastore and `ANALYZE TABLE` only for tables stored as Parquet files and only when stored in the `DFS` storage plugin.
-- The first time you execute ANALYZE TABLE for a table, Drill will scan the entire tables (all files.)
-When you next issue the same command, Drill will scan only those files added since the previous run.
-The command will return the message if table statistics are up-to-date:
+- The first time you execute ANALYZE TABLE for a table, Drill will scan the entire table (all files.)
+When you issue the same command for the next time, Drill will scan only those files added since the previous run.
+The command will return the following message if table statistics are up-to-date:
```
@@ -133,24 +155,24 @@ apache drill (dfs.tmp)> analyze table lineitem refresh metadata;
### Metadata usage
Drill uses the Metastore in several places. When you run a query with multiple directories, files or Parquet row groups,
-Drill will use statistics to "prune" the scan. That is, to identify those directories, files or row groups which
-do not contain data which your query needs. If you add new files or directories, and do not rerun `ANALYZE TABLE`,
-then Drill will assume that existing metadata is invalid and wouldn't use it. Periodically rerun `ANALYZE TABLE` so that
- Drill can use table metadata when possible.
+ Drill will use statistics to "prune" the scan. That is, to identify those directories, files or row groups that
+ do not contain data that your query needs. If you add new files or directories and do not rerun `ANALYZE TABLE`,
+ then Drill will assume that existing metadata is invalid and wouldn't use it. Periodically rerun `ANALYZE TABLE` so
+ that Drill can use table metadata when possible.
### Limitations
-This feature is currently in the alpha phase (preview, experimental) for Drill 1.17 and only applies to Parquet
+This feature is currently in the beta phase (preview, experimental) for Drill 1.17 and only applies to Parquet
tables in this release. You must enable this feature through the `metastore.enabled` system/session option.
## Examples
-Examples throughout this topic use the files and directories described in the following section, Directory and File Setup.
+Examples throughout this topic use the files and directories described in the following section, Directory, and File Setup.
### Directory and File Setup
-Download [TPC-H sf1 tables](https://s3-us-west-1.amazonaws.com/drill-public/tpch/sf1/tpch_sf1_parquet.tar.gz) and unpack
- archive.
+Download [TPC-H sf1 tables](https://s3-us-west-1.amazonaws.com/drill-public/tpch/sf1/tpch_sf1_parquet.tar.gz) and
+ unpack archive.
Create lineitem directory in `/tmp/` and two subdirectories under `/tmp/lineitem` named `s1` and `s2` and copy there table data:
@@ -172,7 +194,7 @@ SELECT count(*) FROM dfs.tmp.lineitem;
1 row selected (0.291 seconds)
```
-Notice that the query plan contains group scan with `usedMetastore = false`:
+Notice that the query plan contains a group scan with `usedMetastore = false`:
```
@@ -183,6 +205,12 @@ Notice that the query plan contains group scan with `usedMetastore = false`:
### Computing and storing table metadata to Drill Metastore
+Enable Drill Metastore:
+
+```
+SET `metastore.enabled` = true;
+```
+
Run [ANALYZE TABLE]({{site.baseurl}}/docs/analyze-table-refresh-metadata) command on the table, whose metadata should
be computed and stored into the Drill Metastore:
@@ -199,7 +227,7 @@ apache drill> ANALYZE TABLE dfs.tmp.lineitem REFRESH METADATA;
The output of this command provides the status of the command execution and its summary.
-Once, its metadata is collected and stored, it will be used when querying the table. To ensure that it was used, its
+Once, table metadata is collected and stored in the Metastore, it will be used when querying the table. To ensure that it was used, its
info was added to the group scan (`usedMetastore=true` entry in `ParquetGroupScan`):
@@ -227,6 +255,8 @@ apache drill> ANALYZE TABLE dfs.tmp.lineitem REFRESH METADATA;
### Exposing Drill Metastore metadata through `INFORMATION_SCHEMA` tables
Drill exposes some Metastore tables metadata through `INFORMATION_SCHEMA` tables.
+Note, that Metastore metadata will be exposed to the info schema, only if Metastore is enabled. If it is disabled, info
+ tables won't contain Metastore metadata.
`TABLES` table includes the set of tables on which you have run `ANALYZE TABLE`.
Description of Metastore-specific columns:
diff --git a/_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md b/_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md
index 1294286..744569b 100644
--- a/_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md
+++ b/_docs/performance-tuning/drill-metastore/030-drill-iceberg-metastore.md
@@ -1,12 +1,18 @@
---
title: "Drill Iceberg Metastore"
parent: "Drill Metastore"
-date:
+date: 2020-01-31
---
-Drill uses Iceberg Metastore implementation based on [Iceberg tables](http://iceberg.incubator.apache.org). For
- details on how to configure Iceberg Metastore implementation and its option descriptions, please refer to
- [Iceberg Metastore docs](https://github.com/apache/drill/blob/master/metastore/iceberg-metastore/README.md).
+Drill uses Iceberg Metastore implementation based on [Iceberg tables](http://iceberg.incubator.apache.org). For Drill 1.17,
+ this is default Drill Metastore implementation. For details on how to configure Iceberg Metastore implementation and
+ its option descriptions, please refer to [Iceberg Metastore docs](https://github.com/apache/drill/blob/master/metastore/iceberg-metastore/README.md).
+
+{% include startnote.html %}
+Iceberg table supports concurrent writes and transactions but they are only effective on file systems that support
+ atomic rename.
+If the file system does not support atomic rename, it could lead to inconsistencies during concurrent writes.
+{% include endnote.html %}
### Iceberg Tables Location