You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by wz...@apache.org on 2022/01/12 21:14:53 UTC
[impala] branch master updated (6894085 -> 79f4c05)
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 6894085 IMPALA-11030: Fix incorrect creation of common partition exprs
new 8ab219a IMPALA-11043: [DOCS] Initial Iceberg documentation
new 79f4c05 IMPALA-10951 (part 2): Upgrade protobuf library to 3.14.0 for Impala
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CMakeLists.txt | 13 +
be/src/kudu/rpc/exactly_once_rpc-test.cc | 6 +-
be/src/kudu/rpc/inbound_call.cc | 4 +-
be/src/kudu/rpc/outbound_call.cc | 6 +-
be/src/kudu/rpc/result_tracker.h | 5 +-
be/src/kudu/rpc/serialization.cc | 8 +-
be/src/kudu/rpc/serialization.h | 2 +-
be/src/kudu/util/pb_util.cc | 56 ++--
be/src/kudu/util/protobuf_util.h | 21 +-
bin/bootstrap_toolchain.py | 3 +
bin/impala-config.sh | 10 +-
docs/impala.ditamap | 1 +
docs/impala_keydefs.ditamap | 5 +
docs/topics/impala_iceberg.xml | 525 +++++++++++++++++++++++++++++++
14 files changed, 617 insertions(+), 48 deletions(-)
create mode 100644 docs/topics/impala_iceberg.xml
[impala] 01/02: IMPALA-11043: [DOCS] Initial Iceberg documentation
Posted by wz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8ab219aa3acd6e1fa3455c94b2aa17bac30c2381
Author: Zoltan Borok-Nagy <bo...@cloudera.com>
AuthorDate: Thu Jan 6 18:51:16 2022 +0100
IMPALA-11043: [DOCS] Initial Iceberg documentation
Impala's Iceberg support has greatly evolved lately. This
commit adds the initial documentation for Iceberg support.
Change-Id: Ief452611319f31d660838d35414bada387a05ef7
Reviewed-on: http://gerrit.cloudera.org:8080/18128
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Tamas Mate <tm...@cloudera.com>
---
docs/impala.ditamap | 1 +
docs/impala_keydefs.ditamap | 5 +
docs/topics/impala_iceberg.xml | 525 +++++++++++++++++++++++++++++++++++++++++
3 files changed, 531 insertions(+)
diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index fe7c954..8e1d4b9 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -328,6 +328,7 @@ under the License.
</topicref>
<topicref href="topics/impala_kudu.xml"/>
<topicref href="topics/impala_hbase.xml"/>
+ <topicref rev="4.1.0" href="topics/impala_iceberg.xml"/>
<topicref href="topics/impala_s3.xml"/>
<topicref rev="2.9.0" href="topics/impala_adls.xml"/>
<topicref href="topics/impala_isilon.xml"/>
diff --git a/docs/impala_keydefs.ditamap b/docs/impala_keydefs.ditamap
index 38da792..6230ac6 100644
--- a/docs/impala_keydefs.ditamap
+++ b/docs/impala_keydefs.ditamap
@@ -53,6 +53,10 @@ under the License.
<topicmeta><linktext>the Apache HBase documentation</linktext></topicmeta>
</keydef>
+ <keydef href="https://iceberg.apache.org" scope="external" format="html" keys="upstream_iceberg_site">
+ <topicmeta><linktext>the Apache Iceberg site</linktext></topicmeta>
+ </keydef>
+
<keydef href="https://hbase.apache.org/book.html#security" scope="external" format="html" keys="upstream_hbase_security_docs">
<topicmeta><linktext>the Security chapter in the Apache HBase documentation</linktext></topicmeta>
</keydef>
@@ -10521,6 +10525,7 @@ under the License.
<keydef href="https://issues.apache.org/jira/browse/IMPALA-9999" scope="external" format="html" keys="IMPALA-9999"/>
<!-- Short form of mapping from Impala release to vendor-specific releases, for use in headings. -->
+ <keydef keys="impala41"><topicmeta><keywords><keyword>Impala 4.1</keyword></keywords></topicmeta></keydef>
<keydef keys="impala40"><topicmeta><keywords><keyword>Impala 4.0</keyword></keywords></topicmeta></keydef>
<keydef keys="impala34"><topicmeta><keywords><keyword>Impala 3.4</keyword></keywords></topicmeta></keydef>
<keydef keys="impala33"><topicmeta><keywords><keyword>Impala 3.3</keyword></keywords></topicmeta></keydef>
diff --git a/docs/topics/impala_iceberg.xml b/docs/topics/impala_iceberg.xml
new file mode 100644
index 0000000..ba33ae3
--- /dev/null
+++ b/docs/topics/impala_iceberg.xml
@@ -0,0 +1,525 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="impala_iceberg">
+
+ <title id="iceberg">Using Impala with Iceberg Tables</title>
+ <titlealts audience="PDF"><navtitle>Iceberg Tables</navtitle></titlealts>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Iceberg"/>
+ <data name="Category" value="Querying"/>
+ <data name="Category" value="Data Analysts"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Tables"/>
+ </metadata>
+ </prolog>
+
+ <conbody>
+
+ <p>
+ <indexterm audience="hidden">Iceberg</indexterm>
+ Impala now supports Apache Iceberg which is an open table format for huge analytic datasets.
+ With this functionality, you can access any existing Iceberg tables using SQL and perform
+ analytics over them. Using Impala you can create and write Iceberg tables in different
+ Iceberg Catalogs (e.g. HiveCatalog, HadoopCatalog). It also supports location-based
+ tables (HadoopTables).
+ </p>
+ <p>
+ Currently only Iceberg V1 DML operations are allowed, i.e. INSERT INTO /INSERT OVERWRITE.
+ Iceberg V2 operations like row-level modifications (UPDATE, DELETE) are not supported yet.
+ </p>
+
+ <p>
+ For more information on Iceberg, see <xref keyref="upstream_iceberg_site"/>.
+ </p>
+
+ <p outputclass="toc inpage"/>
+ </conbody>
+
+ <concept id="iceberg_features">
+ <title>Overview of Iceberg features</title>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Concepts"/>
+ </metadata>
+ </prolog>
+ <conbody>
+ <ul>
+ <li>
+ ACID compliance: DML operations are atomic, queries always read a consistent snapshot.
+ </li>
+ <li>
+ Hidden partitioning: Iceberg produces partition values by taking a column value and
+ optionally transforming it. Partition information is stored in the Iceberg metadata
+ files. Iceberg is able to TRUNCATE column values or calculate
+ a hash of them and use it for partitioning. Readers don't need to be aware of the
+ partitioning of the table.
+ </li>
+ <li>
+ Partition layout evolution: When the data volume or the query patterns change you
+ can update the layout of a table. Since hidden partitioning is used, you don't need to
+ rewrite the data files during partition layout evolution.
+ </li>
+ <li>
+ Schema evolution: supports add, drop, update, or rename schema elements,
+ and has no side-effects.
+ </li>
+ <li>
+ Time travel: enables reproducible queries that use exactly the same table
+ snapshot, or lets users easily examine changes.
+ </li>
+ </ul>
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_create">
+
+ <title>Creating Iceberg tables with Impala</title>
+ <prolog>
+ <metadata>
+ <data name="Category" value="Concepts"/>
+ </metadata>
+ </prolog>
+
+ <conbody>
+ <p>
+ When you have an existing Iceberg table that is not yet present in the Hive Metastore,
+ you can use the <codeph>CREATE EXTERNAL TABLE</codeph> command in Impala to add the table to the Hive
+ Metastore and make Impala able to interact with this table. Currently Impala supports
+ HadoopTables, HadoopCatalog, and HiveCatalog. If you have an existing table in HiveCatalog,
+ and you are using the same Hive Metastore, you need no further actions.
+ </p>
+ <ul>
+ <li>
+ <b>HadoopTables</b>. When the table already exists in a HadoopTable it means there is
+ a location on the file system that contains your table. Use the following command
+ to add this table to Impala's catalog:
+ <codeblock>
+CREATE EXTERNAL TABLE ice_hadoop_tbl
+STORED AS ICEBERG
+LOCATION '/path/to/table'
+TBLPROPERTIES('iceberg.catalog'='hadoop.tables');
+ </codeblock>
+ </li>
+ <li>
+ <b>HadoopCatalog</b>. A table in HadoopCatalog means that there is a catalog location
+ in the file system under which Iceberg tables are stored. Use the following command
+ to add a table in a HadoopCatalog to Impala:
+ <codeblock>
+CREATE EXTERNAL TABLE ice_hadoop_cat
+STORED AS ICEBERG
+TBLPROPERTIES('iceberg.catalog'='hadoop.catalog',
+ 'iceberg.catalog_location'='/path/to/catalog',
+ 'iceberg.table_identifier'='namespace.table');
+ </codeblock>
+ </li>
+ <li>
+ Alternatively, you can also use custom catalogs to use existing tables. It means you need to define
+ your catalog in hive-site.xml.
+ The adventage of this method is that other engines are more likely to be able to interact with this table.
+ To globally register different catalogs, set the following Hadoop configurations:
+ <table rowsep="1" colsep="1" id="iceberg_custom_catalogs">
+ <tgroup cols="2">
+ <colspec colname="c1" colnum="1"/>
+ <colspec colname="c2" colnum="2"/>
+ <thead>
+ <row>
+ <entry>Config Key</entry>
+ <entry>Description</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry>iceberg.catalog.<catalog_name>.type</entry>
+ <entry>type of catalog: hive, hadoop, or left unset if using a custom catalog</entry>
+ </row>
+ <row>
+ <entry>iceberg.catalog.<catalog_name>.catalog-impl</entry>
+ <entry>catalog implementation, must not be null if type is empty</entry>
+ </row>
+ <row>
+ <entry>iceberg.catalog.<catalog_name>.<key></entry>
+ <entry>any config key and value pairs for the catalog</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ <p>
+ For example, to register a HadoopCatalog called 'hadoop', set the following properties in hive-site.xml:
+ <codeblock>
+iceberg.catalog.hadoop.type=hadoop;
+iceberg.catalog.hadoop.warehouse=hdfs://example.com:8020/warehouse;
+ </codeblock>
+ </p>
+ <p>
+ Then in the CREATE TABLE statement you can just refer to the catalog name:
+ <codeblock>
+CREATE EXTERNAL TABLE ice_catalogs STORED AS ICEBERG TBLPROPERTIES('iceberg.catalog'='<CATALOG-NAME>');
+ </codeblock>
+ </p>
+ </li>
+ <li>
+ If the table already exists in HiveCatalog then Impala should be able to see it without any additional
+ commands.
+ </li>
+ </ul>
+
+ <p>
+ You can also create new Iceberg tables with Impala. You can use the same commands as above, just
+ omit the <codeph>EXTERNAL</codeph> keyword. To create an Iceberg table in HiveCatalog the following
+ CREATE TABLE statement can be used:
+ <codeblock>
+CREATE TABLE ice_t (i INT) STORED AS ICEBERG;
+ </codeblock>
+ </p>
+ <p>
+ By default Impala assumes that the Iceberg table uses Parquet data files. ORC is also supported,
+ but we need to tell Impala via setting the table property 'write.format.default' to 'ORC'.
+ </p>
+ <p>
+ You can also use <codeph>CREATE TABLE AS SELECT</codeph> to create new Iceberg tables, e.g.:
+ <codeblock>
+CREATE TABLE ice_ctas STORED AS ICEBERG AS SELECT i, b FROM value_tbl;
+
+CREATE TABLE ice_ctas_part PARTITIONED BY(d) STORED AS ICEBERG AS SELECT s, ts, d FROM value_tbl;
+
+CREATE TABLE ice_ctas_part_spec PARTITIONED BY SPEC (truncate(3, s)) STORED AS ICEBERG AS SELECT cast(t as INT), s, d FROM value_tbl;
+ </codeblock>
+ </p>
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_drop">
+ <title>Dropping Iceberg tables</title>
+ <conbody>
+ <p>
+ One can use <codeph>DROP TABLE</codeph> statement to remove an Iceberg table:
+ <codeblock>
+ DROP TABLE ice_t;
+ </codeblock>
+ </p>
+ <p>
+ When <codeph>external.table.purge</codeph> table property is set to true, then the
+ <codeph>DROP TABLE</codeph> statement will also delete the data files. This property
+ is set to true when Impala creates the Iceberg table via <codeph>CREATE TABLE</codeph>.
+ When <codeph>CREATE EXTERNAL TABLE</codeph> is used (the table already exists in some
+ catalog) then this <codeph>external.table.purge</codeph> is set to false, i.e.
+ <codeph>DROP TABLE</codeph> doesn't remove any files, only the table definition
+ in HMS.
+ </p>
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_types">
+ <title>Supported Data Types for Iceberg Columns</title>
+ <conbody>
+
+ <p>
+ You can get information about the supported Iceberg data tyeps in
+ <xref href="https://iceberg.apache.org/#spec/%23schemas-and-data-types" scope="external" format="html">
+ the Iceberg spec</xref>.
+ </p>
+
+ <p>
+ The Iceberg data types can be mapped to the following SQL types in Impala:
+ <table rowsep="1" colsep="1" id="iceberg_types_sql_types">
+ <tgroup cols="2">
+ <colspec colname="c1" colnum="1"/>
+ <colspec colname="c2" colnum="2"/>
+ <thead>
+ <row>
+ <entry>Iceberg type</entry>
+ <entry>SQL type in Impala</entry>
+ </row>
+ </thead>
+ <tbody>
+ <row>
+ <entry>boolean</entry>
+ <entry>BOOLEAN</entry>
+ </row>
+ <row>
+ <entry>int</entry>
+ <entry>INTEGER</entry>
+ </row>
+ <row>
+ <entry>long</entry>
+ <entry>BIGINT</entry>
+ </row>
+ <row>
+ <entry>float</entry>
+ <entry>FLOAT</entry>
+ </row>
+ <row>
+ <entry>double</entry>
+ <entry>DOUBLE</entry>
+ </row>
+ <row>
+ <entry>decimal(P, S)</entry>
+ <entry>DECIMAL(P, S)</entry>
+ </row>
+ <row>
+ <entry>date</entry>
+ <entry>DATE</entry>
+ </row>
+ <row>
+ <entry>time</entry>
+ <entry>Not supported</entry>
+ </row>
+ <row>
+ <entry>timestamp</entry>
+ <entry>TIMESTAMP</entry>
+ </row>
+ <row>
+ <entry>timestamptz</entry>
+ <entry>Only read support via TIMESTAMP</entry>
+ </row>
+ <row>
+ <entry>string</entry>
+ <entry>STRING</entry>
+ </row>
+ <row>
+ <entry>uuid</entry>
+ <entry>Not supported</entry>
+ </row>
+ <row>
+ <entry>fixed(L)</entry>
+ <entry>Not supported</entry>
+ </row>
+ <row>
+ <entry>binary</entry>
+ <entry>Not supported</entry>
+ </row>
+ <row>
+ <entry>struct</entry>
+ <entry>STRUCT (read only)</entry>
+ </row>
+ <row>
+ <entry>list</entry>
+ <entry>ARRAY (read only)</entry>
+ </row>
+ <row>
+ <entry>map</entry>
+ <entry>MAP (read only)</entry>
+ </row>
+ </tbody>
+ </tgroup>
+ </table>
+ </p>
+ </conbody>
+ </concept>
+
+
+ <concept id="iceberg_schema_evolution">
+ <title>Schema evolution of Iceberg tables</title>
+ <conbody>
+ <p>
+ Iceberg assigns unique field ids to schema elements which means it is possible
+ to reorder/delete/change columns and still be able to correctly read current and
+ old data files. Impala supports the following statements to modify a table's schema:
+ <ul>
+ <li><codeph>ALTER TABLE ... RENAME TO ...</codeph> (renames the table if the Iceberg catalog supports it)</li>
+ <li><codeph>ALTER TABLE ... CHANGE COLUMN ...</codeph> (change name and type of a column iff the new type is compatible with the old type)</li>
+ <li><codeph>ALTER TABLE ... ADD COLUMNS ...</codeph> (adds columns to the end of the table)</li>
+ <li><codeph>ALTER TABLE ... DROP COLUMN ...</codeph></li>
+ </ul>
+ </p>
+ <p>
+ Valid type promotions are:
+ <ul>
+ <li>int to long</li>
+ <li>float to double</li>
+ <li>decimal(P, S) to decimal(P', S) if P' > P – widen the precision of decimal types.</li>
+ </ul>
+ </p>
+ <p>
+ See
+ <xref href="https://iceberg.apache.org/#spec/%23schema-evolution" scope="external" format="html">
+ schema evolution </xref> for more details.
+ </p>
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_partitioning">
+ <title>Partitioning Iceberg tables</title>
+ <conbody>
+ <p>
+ <xref href="https://iceberg.apache.org/#spec/%23partitioning" scope="external" format="html">
+ The Iceberg spec </xref> has information about partitioning Iceberg tables. With Iceberg,
+ we are not limited to value-based partitioning, we can also partition our tables via
+ several partition transforms.
+ </p>
+ <p>
+ Partition transforms are IDENTITY, BUCKET, TRUNCATE, YEAR, MONTH, DAY, HOUR, and VOID.
+ Impala supports all of these transforms. To create a partitioned Iceberg table, one
+ needs to add a <codeph>PARTITIONED BY SPEC</codeph> clause to the CREATE TABLE statement, e.g.:
+ <codeblock>
+CREATE TABLE ice_p (i INT, d DATE, s STRING, t TIMESTAMP)
+PARTITIONED BY SPEC (BUCKET(5, i), MONTH(d), TRUNCATE(3, s), HOUR(t))
+STORED AS ICEBERG;
+ </codeblock>
+ </p>
+ <p>
+ Iceberg also supports
+ <xref href="https://iceberg.apache.org/#spec/%23partition-evolution" scope="external" format="html">
+ partition evolution</xref> which means that the partitioning of a table can be changed, even
+ without the need of rewriting existing data files. You can change an existing table's
+ partitioning via an <codeph>ALTER TABLE SET PARTITION SPEC</codeph> statement, e.g.:
+ <codeblock>
+ALTER TABLE ice_p SET PARTITION SPEC (VOID(i), VOID(d), TRUNCATE(3, s), HOUR(t), i);
+ </codeblock>
+ </p>
+ <p>
+ Please keep in mind that for Iceberg V1 tables:
+ <ul>
+ <li>Do not reorder partition fields</li>
+ <li>Do not drop partition fields; instead replace the field’s transform with the void transform</li>
+ <li>Only add partition fields at the end of the previous partition spec</li>
+ </ul>
+ </p>
+ <p>
+ You can also use the legacy syntax to create identity-partitioned Iceberg tables:
+ <codeblock>
+CREATE TABLE ice_p (i INT, b INT) PARTITIONED BY (p1 INT, p2 STRING) STORED AS ICEBERG;
+ </codeblock>
+ </p>
+ <p>
+ One can inspect a table's partition spec by the <codeph>SHOW PARTITIONS</codeph> or
+ <codeph>SHOW CREATE TABLE</codeph> commands.
+ </p>
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_inserts">
+ <title>Writing Iceberg tables</title>
+ <conbody>
+ <p>
+ Impala is also able to insert new data to Iceberg tables. Currently the <codeph>INSERT INTO</codeph>
+ and <codeph>INSERT OVERWRITE</codeph> DML statements are supported. One can also remove the
+ contents of an Iceberg table via the <codeph>TRUNCATE</codeph> command.
+ </p>
+ <p>
+ Since Iceberg uses hidden partitioning it means you don't need a partition clause in your INSERT
+ statements. E.g. insertion to a partitioned table looks like:
+ <codeblock>
+CREATE TABLE ice_p (i INT, b INT) PARTITIONED BY SPEC (bucket(17, i)) STORED AS ICEBERG;
+INSERT INTO ice_p VALUES (1, 2);
+ </codeblock>
+ </p>
+ <p>
+ <codeph>INSERT OVERWRITE</codeph> statements can replace data in the table with the result of a query.
+ For partitioned tables Impala does a dynamic overwrite, which means partitions that have rows produced
+ by the SELECT query will be replaced. And partitions that have no rows produced by the SELECT query
+ remain untouched. INSERT OVERWRITE is not allowed for tables that use the BUCKET partition transform
+ because dynamic overwrite behavior would be too random in this case. If one needs to replace all
+ contents of a table, they can still use <codeph>TRUNCATE</codeph> and <codeph>INSERT INTO</codeph>.
+ </p>
+ <p>
+ Impala can only write Iceberg tables with Parquet data files.
+ </p>
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_time_travel">
+ <title>Time travel for Iceberg tables</title>
+ <conbody>
+
+ <p>
+ Iceberg stores the table states in a chain of snapshots. By default, Impala uses the current
+ snapshot of the table. But for Iceberg tables, it is also possible to query an earlier state of
+ the table.
+ </p>
+
+ <p>
+ We can use the <codeph>FOR SYSTEM_TIME AS OF</codeph> and <codeph>FOR SYSTEM_VERSION AS OF</codeph>
+ clauses in <codeph>SELECT</codeph> queries, e.g.:
+ <codeblock>
+SELECT * FROM ice_t FOR SYSTEM_TIME AS OF '2022-01-04 10:00:00';
+SELECT * FROM ice_t FOR SYSTEM_TIME AS OF now() - interval 5 days;
+SELECT * FROM ice_t FOR SYSTEM_VERSION AS OF 123456;
+ </codeblock>
+ </p>
+
+ <p>
+ If one needs to check the available snapshots of a table they can use the <codeph>DESCRIBE HISTORY</codeph>
+ statement:
+ <codeblock>
+DESCRIBE HISTORY ice_t;
+ </codeblock>
+ </p>
+
+ <p>
+ Please note that during time travel, Impala uses the current table schema to query an older
+ snapshot of the table which might have had a different schema in the past.
+ </p>
+
+ </conbody>
+ </concept>
+
+ <concept id="iceberg_table_properties">
+ <title>Iceberg table properties</title>
+ <conbody>
+ <p>
+ We can set the following table properties for Iceberg tables:
+ <ul>
+ <li>
+ <codeph>iceberg.catalog</codeph>: controls which catalog is used for this Iceberg table.
+ It can be 'hive.catalog' (default), 'hadoop.catalog', 'hadoop.tables', or a name that
+ identifies a catalog defined in the Hadoop configurations, e.g. hive-site.xml
+ </li>
+ <li><codeph>iceberg.catalog_location</codeph>: Iceberg table catalog location when <codeph>iceberg.catalog</codeph> is <codeph>'hadoop.catalog'</codeph></li>
+ <li><codeph>iceberg.table_identifier</codeph>: Iceberg table identifier. We use <database>.<table> instead if this property is not set</li>
+ <li><codeph>write.format.default</codeph>: data file format of the table. Impala can read ORC and PARQUET data files in Iceberg tables, and can write PARQUET data files only.</li>
+ <li><codeph>write.parquet.compression-codec</codeph>:
+ Parquet compression codec. Supported values are: NONE, GZIP, SNAPPY
+ (default value), LZ4, ZSTD. The table property will be ignored if
+ <codeph>COMPRESSION_CODEC</codeph> query option is set.
+ </li>
+ <li><codeph>write.parquet.compression-level</codeph>:
+ Parquet compression level. Used with ZSTD compression only.
+ Supported range is [1, 22]. Default value is 3. The table property
+ will be ignored if <codeph>COMPRESSION_CODEC</codeph> query option is set.
+ </li>
+ <li><codeph>write.parquet.row-group-size-bytes</codeph>:
+ Parquet row group size in bytes. Supported range is [8388608,
+ 2146435072] (8MB - 2047MB). The table property will be ignored if
+ <codeph>PARQUET_FILE_SIZE</codeph> query option is set.
+ If neither the table property nor the <codeph>PARQUET_FILE_SIZE</codeph> query option
+ is set, the way Impala calculates row group size will remain
+ unchanged.
+ </li>
+ <li><codeph>write.parquet.page-size-bytes</codeph>:
+ Parquet page size in bytes. Used for PLAIN encoding. Supported range
+ is [65536, 1073741824] (64KB - 1GB).
+ If the table property is unset, the way Impala calculates page size
+ will remain unchanged.
+ </li>
+ <li><codeph>write.parquet.dict-size-bytes</codeph>:
+ Parquet dictionary page size in bytes. Used for dictionary encoding.
+ Supported range is [65536, 1073741824] (64KB - 1GB).
+ If the table property is unset, the way Impala calculates dictionary
+ page size will remain unchanged.
+ </li>
+ </ul>
+ </p>
+ </conbody>
+ </concept>
+</concept>
[impala] 02/02: IMPALA-10951 (part 2): Upgrade protobuf library to 3.14.0 for Impala
Posted by wz...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 79f4c0576c857c3814bee52b1763c3db855c521b
Author: wzhou-code <wz...@cloudera.com>
AuthorDate: Thu Oct 28 00:21:24 2021 -0700
IMPALA-10951 (part 2): Upgrade protobuf library to 3.14.0 for Impala
There are some API changes in the new version of protobuf library.
This patch makes necessary changes for Impala code to pass compiling.
Two tarballs of protobuf 3.14.0 are built in toolchain. Use
protobuf-3.14.0 for normal builds, use protobuf-3.14.0-clangcompat-p2
for Clang builds.
Bump up Kudu to the latest version 67ba3cae45.
Testing:
- Passed core DEBUG build and exhaustive release build.
- Passed core ASAN build.
Change-Id: Ia1df4faceff9fda169c9d15fe8b1e69cfabe0d43
Reviewed-on: http://gerrit.cloudera.org:8080/17948
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
CMakeLists.txt | 13 ++++++++
be/src/kudu/rpc/exactly_once_rpc-test.cc | 6 ++--
be/src/kudu/rpc/inbound_call.cc | 4 ++-
be/src/kudu/rpc/outbound_call.cc | 6 ++--
be/src/kudu/rpc/result_tracker.h | 5 +--
be/src/kudu/rpc/serialization.cc | 8 ++---
be/src/kudu/rpc/serialization.h | 2 +-
be/src/kudu/util/pb_util.cc | 56 +++++++++++++++++++-------------
be/src/kudu/util/protobuf_util.h | 21 +++++++-----
bin/bootstrap_toolchain.py | 3 ++
bin/impala-config.sh | 10 +++---
11 files changed, 86 insertions(+), 48 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 566c884..3ca77fb 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -112,6 +112,8 @@ set_dep_root(LZ4)
set_dep_root(ZSTD)
set_dep_root(OPENLDAP)
set_dep_root(PROTOBUF)
+set(PROTOBUF_CLANG_ROOT
+ $ENV{IMPALA_TOOLCHAIN_PACKAGES_HOME}/protobuf-$ENV{IMPALA_PROTOBUF_CLANG_VERSION})
set_dep_root(RE2)
set_dep_root(RAPIDJSON)
set_dep_root(SNAPPY)
@@ -315,6 +317,17 @@ find_package(Cctz REQUIRED)
IMPALA_ADD_THIRDPARTY_LIB(cctz ${CCTZ_INCLUDE_DIR} ${CCTZ_STATIC_LIB} "")
# find protobuf headers, libs and compiler
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "ADDRESS_SANITIZER"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TIDY"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN_FULL"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN"
+ OR "${CMAKE_BUILD_TYPE}" STREQUAL "TSAN_FULL")
+ # Use the protobuf library with patches for Clang builds.
+ set(PROTOBUF_ROOT ${PROTOBUF_CLANG_ROOT})
+endif()
+message(STATUS "PROTOBUF_ROOT: " ${PROTOBUF_ROOT})
+
find_package(Protobuf REQUIRED)
IMPALA_ADD_THIRDPARTY_LIB(protobuf ${PROTOBUF_INCLUDE_DIR} ${PROTOBUF_STATIC_LIBRARY}
${PROTOBUF_SHARED_LIBRARY})
diff --git a/be/src/kudu/rpc/exactly_once_rpc-test.cc b/be/src/kudu/rpc/exactly_once_rpc-test.cc
index c94e89c..638072b 100644
--- a/be/src/kudu/rpc/exactly_once_rpc-test.cc
+++ b/be/src/kudu/rpc/exactly_once_rpc-test.cc
@@ -390,7 +390,7 @@ class ExactlyOnceRpcTest : public RpcTestBase {
TEST_F(ExactlyOnceRpcTest, TestExactlyOnceSemanticsAfterRpcCompleted) {
ASSERT_OK(StartServer());
ExactlyOnceResponsePB original_resp;
- int mem_consumption = mem_tracker_->consumption();
+ size_t mem_consumption = mem_tracker_->consumption();
{
RpcController controller;
ExactlyOnceRequestPB req;
@@ -404,9 +404,9 @@ TEST_F(ExactlyOnceRpcTest, TestExactlyOnceSemanticsAfterRpcCompleted) {
// The incremental usage of a new client is the size of the response itself
// plus some fixed overhead for the client-tracking structure.
- int expected_incremental_usage = original_resp.SpaceUsed() + 200;
+ size_t expected_incremental_usage = original_resp.SpaceUsedLong() + 200;
- int mem_consumption_after = mem_tracker_->consumption();
+ size_t mem_consumption_after = mem_tracker_->consumption();
ASSERT_GT(mem_consumption_after - mem_consumption, expected_incremental_usage);
mem_consumption = mem_consumption_after;
}
diff --git a/be/src/kudu/rpc/inbound_call.cc b/be/src/kudu/rpc/inbound_call.cc
index 7479020..edb1d3c 100644
--- a/be/src/kudu/rpc/inbound_call.cc
+++ b/be/src/kudu/rpc/inbound_call.cc
@@ -18,6 +18,7 @@
#include "kudu/rpc/inbound_call.h"
#include <cstdint>
+#include <limits>
#include <memory>
#include <ostream>
@@ -177,7 +178,8 @@ void InboundCall::SerializeResponseBuffer(const MessageLite& response,
// happened.
}
- uint32_t protobuf_msg_size = response.ByteSize();
+ size_t protobuf_msg_size = response.ByteSizeLong();
+ CHECK_LE(protobuf_msg_size, std::numeric_limits<uint32_t>::max());
ResponseHeader resp_hdr;
resp_hdr.set_call_id(header_.call_id());
diff --git a/be/src/kudu/rpc/outbound_call.cc b/be/src/kudu/rpc/outbound_call.cc
index 3fe7641..11ca221 100644
--- a/be/src/kudu/rpc/outbound_call.cc
+++ b/be/src/kudu/rpc/outbound_call.cc
@@ -18,6 +18,8 @@
#include "kudu/rpc/outbound_call.h"
#include <cstdint>
+#include <functional>
+#include <limits>
#include <memory>
#include <mutex>
#include <string>
@@ -25,7 +27,6 @@
#include <utility>
#include <vector>
-#include <boost/function.hpp>
#include <gflags/gflags.h>
#include <google/protobuf/message.h>
@@ -147,7 +148,8 @@ void OutboundCall::SetRequestPayload(const Message& req,
// Compute total size of sidecar payload so that extra space can be reserved as part of
// the request body.
- uint32_t message_size = req.ByteSize();
+ size_t message_size = req.ByteSizeLong();
+ CHECK_LE(message_size, std::numeric_limits<uint32_t>::max());
sidecar_byte_size_ = 0;
for (const unique_ptr<RpcSidecar>& car: sidecars_) {
header_.add_sidecar_offsets(sidecar_byte_size_ + message_size);
diff --git a/be/src/kudu/rpc/result_tracker.h b/be/src/kudu/rpc/result_tracker.h
index 890bd74..f095728 100644
--- a/be/src/kudu/rpc/result_tracker.h
+++ b/be/src/kudu/rpc/result_tracker.h
@@ -282,8 +282,9 @@ class ResultTracker : public RefCountedThreadSafe<ResultTracker> {
// Calculates the memory footprint of this struct.
int64_t memory_footprint() const {
return kudu_malloc_usable_size(this)
- + (ongoing_rpcs.capacity() > 0 ? kudu_malloc_usable_size(ongoing_rpcs.data()) : 0)
- + (response.get() != nullptr ? response->SpaceUsed() : 0);
+ + (ongoing_rpcs.capacity() > 0 ? kudu_malloc_usable_size(ongoing_rpcs.data()) :
+ 0)
+ + (response ? response->SpaceUsedLong() : 0);
}
};
diff --git a/be/src/kudu/rpc/serialization.cc b/be/src/kudu/rpc/serialization.cc
index 78ea295..28c015b 100644
--- a/be/src/kudu/rpc/serialization.cc
+++ b/be/src/kudu/rpc/serialization.cc
@@ -56,8 +56,8 @@ enum {
void SerializeMessage(const MessageLite& message, faststring* param_buf,
int additional_size, bool use_cached_size) {
DCHECK_GE(additional_size, 0);
- int pb_size = use_cached_size ? message.GetCachedSize() : message.ByteSize();
- DCHECK_EQ(message.ByteSize(), pb_size);
+ size_t pb_size = use_cached_size ? message.GetCachedSize() : message.ByteSizeLong();
+ DCHECK_EQ(message.ByteSizeLong(), pb_size);
// Use 8-byte integers to avoid overflowing when additional_size approaches INT_MAX.
int64_t recorded_size = static_cast<int64_t>(pb_size) +
static_cast<int64_t>(additional_size);
@@ -91,7 +91,7 @@ void SerializeHeader(const MessageLite& header,
<< "RPC header missing fields: " << header.InitializationErrorString();
// Compute all the lengths for the packet.
- size_t header_pb_len = header.ByteSize();
+ size_t header_pb_len = header.ByteSizeLong();
size_t header_tot_len = kMsgLengthPrefixLength // Int prefix for the total length.
+ CodedOutputStream::VarintSize32(header_pb_len) // Varint delimiter for header PB.
+ header_pb_len; // Length for the header PB itself.
@@ -137,7 +137,7 @@ Status ParseMessage(const Slice& buf,
// Protobuf enforces a 64MB total bytes limit on CodedInputStream by default.
// Override this default with the actual size of the buffer to allow messages
// larger than 64MB.
- in.SetTotalBytesLimit(buf.size(), -1);
+ in.SetTotalBytesLimit(buf.size());
in.Skip(kMsgLengthPrefixLength);
uint32_t header_len;
diff --git a/be/src/kudu/rpc/serialization.h b/be/src/kudu/rpc/serialization.h
index 9fa3858..b1d7c84 100644
--- a/be/src/kudu/rpc/serialization.h
+++ b/be/src/kudu/rpc/serialization.h
@@ -43,7 +43,7 @@ namespace serialization {
// the protobuf itself).
// 'use_cached_size' Additional optional argument whether to use the cached
// or explicit byte size by calling MessageLite::GetCachedSize() or
-// MessageLite::ByteSize(), respectively.
+// MessageLite::ByteSizeLong(), respectively.
// Out: The faststring 'param_buf' to be populated with the serialized bytes.
// The faststring's length is only determined by the amount that
// needs to be serialized for the protobuf (i.e., no additional space
diff --git a/be/src/kudu/util/pb_util.cc b/be/src/kudu/util/pb_util.cc
index 6237c80..7461b4f 100644
--- a/be/src/kudu/util/pb_util.cc
+++ b/be/src/kudu/util/pb_util.cc
@@ -27,6 +27,7 @@
#include <cstddef>
#include <deque>
#include <initializer_list>
+#include <limits>
#include <memory>
#include <mutex>
#include <ostream>
@@ -140,9 +141,8 @@ namespace {
// protobuf implementation but is more likely caused by concurrent modification
// of the message. This function attempts to distinguish between the two and
// provide a useful error message.
-void ByteSizeConsistencyError(int byte_size_before_serialization,
- int byte_size_after_serialization,
- int bytes_produced_by_serialization) {
+void ByteSizeConsistencyError(size_t byte_size_before_serialization,
+ size_t byte_size_after_serialization, size_t bytes_produced_by_serialization) {
CHECK_EQ(byte_size_before_serialization, byte_size_after_serialization)
<< "Protocol message was modified concurrently during serialization.";
CHECK_EQ(bytes_produced_by_serialization, byte_size_before_serialization)
@@ -365,7 +365,7 @@ Status ReadPBStartingAt(ReadableFileType* reader, int version,
// integer overflow warnings, so that's what we'll use.
ArrayInputStream ais(body.data(), body.size());
CodedInputStream cis(&ais);
- cis.SetTotalBytesLimit(512 * 1024 * 1024, -1);
+ cis.SetTotalBytesLimit(512 * 1024 * 1024);
if (PREDICT_FALSE(!msg->ParseFromCodedStream(&cis))) {
return Status::IOError("Unable to parse PB from path", reader->filename());
}
@@ -471,16 +471,16 @@ void AppendToString(const MessageLite &msg, faststring *output) {
void AppendPartialToString(const MessageLite &msg, faststring* output) {
size_t old_size = output->size();
- int byte_size = msg.ByteSize();
+ size_t byte_size = msg.ByteSizeLong();
// Messages >2G cannot be serialized due to overflow computing ByteSize.
DCHECK_GE(byte_size, 0) << "Error computing ByteSize";
- output->resize(old_size + static_cast<size_t>(byte_size));
+ output->resize(old_size + byte_size);
uint8* start = &((*output)[old_size]);
uint8* end = msg.SerializeWithCachedSizesToArray(start);
if (end - start != byte_size) {
- ByteSizeConsistencyError(byte_size, msg.ByteSize(), end - start);
+ ByteSizeConsistencyError(byte_size, msg.ByteSizeLong(), end - start);
}
}
@@ -593,31 +593,40 @@ void TruncateFields(Message* message, int max_len) {
}
namespace {
-class SecureFieldPrinter : public TextFormat::FieldValuePrinter {
+class SecureFieldPrinter : public TextFormat::FastFieldValuePrinter {
public:
- using super = TextFormat::FieldValuePrinter;
+ using super = TextFormat::FastFieldValuePrinter;
+ using BaseTextGenerator = TextFormat::BaseTextGenerator;
- string PrintFieldName(const Message& message,
- const Reflection* reflection,
- const FieldDescriptor* field) const override {
+ void PrintFieldName(const Message& message, const Reflection* reflection,
+ const FieldDescriptor* field, BaseTextGenerator* generator) const override {
hide_next_string_ = field->cpp_type() == FieldDescriptor::CPPTYPE_STRING &&
field->options().GetExtension(REDACT);
- return super::PrintFieldName(message, reflection, field);
+ super::PrintFieldName(message, reflection, field, generator);
}
- string PrintString(const string& val) const override {
+ void PrintFieldName(const Message& message, int field_index, int field_count,
+ const Reflection* reflection, const FieldDescriptor* field,
+ BaseTextGenerator* generator) const override {
+ hide_next_string_ = field->cpp_type() == FieldDescriptor::CPPTYPE_STRING
+ && field->options().GetExtension(REDACT);
+ super::PrintFieldName(
+ message, field_index, field_count, reflection, field, generator);
+ }
+ void PrintString(const string& val, BaseTextGenerator* generator) const override {
if (hide_next_string_) {
hide_next_string_ = false;
- return KUDU_REDACT(super::PrintString(val));
+ super::PrintString(KUDU_REDACT(val), generator);
+ return;
}
- return super::PrintString(val);
+ super::PrintString(val, generator);
}
- string PrintBytes(const string& val) const override {
+ void PrintBytes(const string& val, BaseTextGenerator* generator) const override {
if (hide_next_string_) {
hide_next_string_ = false;
- return KUDU_REDACT(super::PrintBytes(val));
+ super::PrintBytes(KUDU_REDACT(val), generator);
}
- return super::PrintBytes(val);
+ super::PrintBytes(val, generator);
}
mutable bool hide_next_string_ = false;
@@ -774,9 +783,10 @@ const string& WritablePBContainerFile::filename() const {
Status WritablePBContainerFile::AppendMsgToBuffer(const Message& msg, faststring* buf) {
DCHECK(msg.IsInitialized()) << InitializationErrorMessage("serialize", msg);
- int data_len = msg.ByteSize();
- // Messages >2G cannot be serialized due to overflow computing ByteSize.
- DCHECK_GE(data_len, 0) << "Error computing ByteSize";
+ size_t data_len_long = msg.ByteSizeLong();
+ // Messages >2G cannot be serialized due to format restrictions.
+ CHECK_LT(data_len_long, std::numeric_limits<int32_t>::max());
+ uint32_t data_len = static_cast<uint32_t>(data_len_long);
uint64_t record_buflen = sizeof(uint32_t) + data_len + sizeof(uint32_t);
if (version_ >= 2) {
record_buflen += sizeof(uint32_t); // Additional checksum just for the length.
@@ -789,7 +799,7 @@ Status WritablePBContainerFile::AppendMsgToBuffer(const Message& msg, faststring
// Serialize the data length.
size_t cur_offset = 0;
- InlineEncodeFixed32(dst + cur_offset, static_cast<uint32_t>(data_len));
+ InlineEncodeFixed32(dst + cur_offset, data_len);
cur_offset += sizeof(uint32_t);
// For version >= 2: Serialize the checksum of the data length.
diff --git a/be/src/kudu/util/protobuf_util.h b/be/src/kudu/util/protobuf_util.h
index cc88eda..b9452d9 100644
--- a/be/src/kudu/util/protobuf_util.h
+++ b/be/src/kudu/util/protobuf_util.h
@@ -17,20 +17,25 @@
#ifndef KUDU_UTIL_PROTOBUF_UTIL_H
#define KUDU_UTIL_PROTOBUF_UTIL_H
+#include <cstdint>
+
+#include <glog/logging.h>
#include <google/protobuf/message_lite.h>
+#include "kudu/util/faststring.h"
+
namespace kudu {
-bool AppendPBToString(const google::protobuf::MessageLite &msg, faststring *output) {
- int old_size = output->size();
- int byte_size = msg.ByteSize();
+inline bool AppendPBToString(
+ const google::protobuf::MessageLite& msg, faststring* output) {
+ size_t old_size = output->size();
+ size_t byte_size = msg.ByteSizeLong();
output->resize(old_size + byte_size);
- uint8* start = reinterpret_cast<uint8*>(output->data() + old_size);
- uint8* end = msg.SerializeWithCachedSizesToArray(start);
+ uint8_t* start = reinterpret_cast<uint8_t*>(output->data() + old_size);
+ uint8_t* end = msg.SerializeWithCachedSizesToArray(start);
CHECK(end - start == byte_size)
- << "Error in serialization. byte_size=" << byte_size
- << " new ByteSize()=" << msg.ByteSize()
- << " end-start=" << (end-start);
+ << "Error in serialization. byte_size=" << byte_size
+ << " new ByteSizeLong()=" << msg.ByteSizeLong() << " end-start=" << (end - start);
return true;
}
diff --git a/bin/bootstrap_toolchain.py b/bin/bootstrap_toolchain.py
index bedb9c2..6624c66 100755
--- a/bin/bootstrap_toolchain.py
+++ b/bin/bootstrap_toolchain.py
@@ -468,6 +468,9 @@ def get_toolchain_downloads():
"jwt-cpp", "libev", "libunwind", "lz4", "openldap", "openssl", "orc", "protobuf",
"python", "rapidjson", "re2", "snappy", "thrift", "tpc-h", "tpc-ds", "zlib",
"zstd"])
+ protobuf_package_clang = ToolchainPackage(
+ "protobuf", explicit_version=os.environ.get("IMPALA_PROTOBUF_CLANG_VERSION"))
+ toolchain_packages += [protobuf_package_clang]
# Check whether this platform is supported (or whether a valid custom toolchain
# has been provided).
if not try_get_platform_release_label() \
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index d34f3a2..9e19c5f 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -68,7 +68,7 @@ fi
# moving to a different build of the toolchain, e.g. when a version is bumped or a
# compile option is changed. The build id can be found in the output of the toolchain
# build jobs, it is constructed from the build number and toolchain git hash prefix.
-export IMPALA_TOOLCHAIN_BUILD_ID=70-b2f92bd26e
+export IMPALA_TOOLCHAIN_BUILD_ID=81-db38b5fb38
# Versions of toolchain dependencies.
# -----------------------------------
export IMPALA_AVRO_VERSION=1.7.4-p5
@@ -136,10 +136,12 @@ export IMPALA_OPENLDAP_VERSION=2.4.47
unset IMPALA_OPENLDAP_URL
export IMPALA_OPENSSL_VERSION=1.0.2l
unset IMPALA_OPENSSL_URL
-export IMPALA_ORC_VERSION=1.7.0-p4
+export IMPALA_ORC_VERSION=1.7.0-p5
unset IMPALA_ORC_URL
-export IMPALA_PROTOBUF_VERSION=3.5.1
+export IMPALA_PROTOBUF_VERSION=3.14.0
unset IMPALA_PROTOBUF_URL
+export IMPALA_PROTOBUF_CLANG_VERSION=3.14.0-clangcompat-p2
+unset IMPALA_PROTOBUF_CLANG_URL
export IMPALA_POSTGRES_JDBC_DRIVER_VERSION=42.2.14
unset IMPALA_POSTGRES_JDBC_DRIVER_URL
export IMPALA_PYTHON_VERSION=2.7.16
@@ -729,7 +731,7 @@ fi
# overall build type) and does not apply when using a local Kudu build.
export USE_KUDU_DEBUG_BUILD=${USE_KUDU_DEBUG_BUILD-false}
-export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"1754f517b2"}
+export IMPALA_KUDU_VERSION=${IMPALA_KUDU_VERSION-"67ba3cae45"}
export IMPALA_KUDU_HOME=${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-$IMPALA_KUDU_VERSION
export IMPALA_KUDU_JAVA_HOME=\
${IMPALA_TOOLCHAIN_PACKAGES_HOME}/kudu-${IMPALA_KUDU_VERSION}/java