You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/07/28 13:09:32 UTC
[impala] branch master updated (efc303b71 -> 74fcacc17)
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from efc303b71 IMPALA-11434: Fix analysis of multiple more than 1d arrays in select list
new b157f1a01 IMPALA-11451: Add net-tools dependency to bootstrap_system.sh
new 614dc54dc IMPALA-11446: Push-down NOT_IN predicate to iceberg
new cbd3fab8c IMPALA-11268: Allow STORED BY and STORED AS as well
new e00a2cbd8 IMPALA-11380: Fix trailing whitespace for VerticalOutputFormatter
new 74fcacc17 IMPALA-11354: Fix load_data.py can't force reload views
The 5 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/bootstrap_system.sh | 5 +++--
fe/src/main/cup/sql-parser.cup | 10 +++++++++
.../org/apache/impala/planner/IcebergScanNode.java | 14 +++++++-----
.../org/apache/impala/analysis/AnalyzeDDLTest.java | 9 ++++++++
.../org/apache/impala/analysis/ParserTest.java | 7 ++++++
shell/shell_output.py | 9 +++++++-
testdata/bin/generate-schema-statements.py | 6 ++++-
.../queries/QueryTest/iceberg-create.test | 13 +++++++++++
.../QueryTest/iceberg-in-predicate-push-down.test | 26 ++++++++++++++++++++++
.../queries/QueryTest/kudu_create.test | 13 +++++++++++
tests/shell/test_shell_commandline.py | 5 ++++-
11 files changed, 106 insertions(+), 11 deletions(-)
[impala] 03/05: IMPALA-11268: Allow STORED BY and STORED AS as well
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit cbd3fab8c48a965dbda9d23bbe75a665ef0da2e7
Author: Gergely Fürnstáhl <gf...@cloudera.com>
AuthorDate: Mon Jul 18 16:08:47 2022 +0200
IMPALA-11268: Allow STORED BY and STORED AS as well
Extended the parser to support 'stored by' keyword for storage
engines, namely Kudu and Iceberg at the moment, to match hive's
syntax. Furthermore, this lays the ground work for seperating
storage engines from file formats to be able specify both with
"STORED BY ... STORED AS ...".
Testing:
Added front-end Parser and Analyzer tests and query tests for table
creation.
Change-Id: Ib677bea8e582bbc01c5fb8c81df57eb60b0ed961
Reviewed-on: http://gerrit.cloudera.org:8080/18743
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
fe/src/main/cup/sql-parser.cup | 10 ++++++++++
.../java/org/apache/impala/analysis/AnalyzeDDLTest.java | 9 +++++++++
fe/src/test/java/org/apache/impala/analysis/ParserTest.java | 7 +++++++
.../functional-query/queries/QueryTest/iceberg-create.test | 13 +++++++++++++
.../functional-query/queries/QueryTest/kudu_create.test | 13 +++++++++++++
5 files changed, 52 insertions(+)
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 0fa7b1b5a..a2afeced6 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -514,6 +514,7 @@ nonterminal Boolean external_val;
nonterminal Boolean purge_val;
nonterminal String opt_init_string_val;
nonterminal THdfsFileFormat file_format_val;
+nonterminal THdfsFileFormat storage_engine_val;
nonterminal THdfsFileFormat file_format_create_table_val;
nonterminal Boolean if_exists_val;
nonterminal Boolean if_not_exists_val;
@@ -2023,6 +2024,8 @@ terminator_val ::=
file_format_create_table_val ::=
KW_STORED KW_AS file_format_val:file_format
{: RESULT = file_format; :}
+ | KW_STORED KW_BY storage_engine_val:storage_engine
+ {: RESULT = storage_engine; :}
|
{: RESULT = null; :}
;
@@ -2052,6 +2055,13 @@ file_format_val ::=
{: RESULT = THdfsFileFormat.JSON; :}
;
+storage_engine_val ::=
+ KW_KUDU
+ {: RESULT = THdfsFileFormat.KUDU; :}
+ | KW_ICEBERG
+ {: RESULT = THdfsFileFormat.ICEBERG; :}
+ ;
+
tbl_properties ::=
KW_TBLPROPERTIES LPAREN properties_map:map RPAREN
{: RESULT = map; :}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index a473a2011..e8c8e4778 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -2313,6 +2313,15 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"Unable to infer the column type for column 'new_col'. Use cast() to " +
"explicitly specify the column type for column 'new_col'.");
+ // IMPALA-11268: Allow STORED BY as well for storage engines
+ AnalyzesOk("create table t primary key (id) " +
+ " stored by kudu as select id, bool_col, tinyint_col, smallint_col, " +
+ "int_col, bigint_col, float_col, double_col, date_string_col, string_col " +
+ "from functional.alltypestiny");
+ AnalyzesOk("create table t primary key (id) stored by iceberg as select id, " +
+ "bool_col, int_col, float_col, double_col, date_string_col, string_col " +
+ "from functional.alltypestiny");
+
// IMPALA-9822 Row Format Delimited is valid only for Text Files
String[] fileFormats = {"PARQUET", "ICEBERG"};
for (String format : fileFormats) {
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index c8d0419a3..48b64ec31 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -2798,6 +2798,13 @@ public class ParserTest extends FrontendTestBase {
ParserError("CREATE TABLE foo (PRIMARY KEY(a), a INT) STORED AS KUDU");
ParserError("CREATE TABLE foo (i INT) PRIMARY KEY (i) STORED AS KUDU");
+ // Supported storage engines
+ ParsesOk("CREATE TABLE foo (i INT) STORED BY KUDU");
+ ParsesOk("CREATE TABLE foo (i INT) STORED BY ICEBERG");
+ ParserError("CREATE TABLE foo (i INT) STORED BY PARQUET");
+ ParserError("CREATE TABLE foo (i INT) STORED BY FOOBAR");
+ ParserError("CREATE TABLE foo (i INT) STORED BY");
+
// Primary key and foreign key specification.
ParsesOk("create table foo(id int, year int, primary key (id))");
ParsesOk("create table foo(id int, year int, primary key (id, year))");
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
index 630809892..1f44026e7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-create.test
@@ -554,3 +554,16 @@ describe formatted ice_non_partition;
---- TYPES
string, string, string
====
+---- QUERY
+# Create with keyword 'stored by'
+create table iceberg_stored_by (i int, s string, ts timestamp, d date) stored by iceberg;
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+describe formatted iceberg_stored_by;
+---- RESULTS: VERIFY_IS_SUBSET
+'','table_type ','ICEBERG '
+---- TYPES
+string, string, string
+====
\ No newline at end of file
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
index c0034357d..7c151559f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_create.test
@@ -470,3 +470,16 @@ select * from ctas_varchar;
---- TYPES
STRING
====
+---- QUERY
+# Create with keyword 'stored by'
+create table kudu_stored_by (i int primary key, s string, ts timestamp, d date) stored by kudu;
+---- RESULTS
+'Table has been created.'
+====
+---- QUERY
+describe formatted kudu_stored_by;
+---- RESULTS: VERIFY_IS_SUBSET
+'','storage_handler ','org.apache.hadoop.hive.kudu.KuduStorageHandler'
+---- TYPES
+string, string, string
+====
\ No newline at end of file
[impala] 05/05: IMPALA-11354: Fix load_data.py can't force reload views
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 74fcacc17bc8aefd824babc45eb8093d3ae2698e
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Fri Jun 24 17:40:58 2022 +0800
IMPALA-11354: Fix load_data.py can't force reload views
When force reloading a test table, we drop it before recreate it. We do
the same for views. However, the DROP statement is "DROP TABLE ..."
which can't be executed on views. This causes the view not actually
dropped and the new view is not created as a consequence. The view
remains unchanged.
This patch fixes generate-schema-statements.py to generate DROP VIEW
statement when force reloading views.
Tested locally by force reloading functional.alltypes_view. Verified the
statements succeed in the sql log file.
Change-Id: I0501fe134b83a19ff09d83bc8d87fdeded860244
Reviewed-on: http://gerrit.cloudera.org:8080/18666
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
testdata/bin/generate-schema-statements.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 00bc9bf6a..525f80f0f 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -240,7 +240,11 @@ def build_create_statement(table_template, table_name, db_name, db_suffix,
force_reload):
create_stmt = ''
if (force_reload):
- create_stmt += 'DROP TABLE IF EXISTS %s%s.%s;\n' % (db_name, db_suffix, table_name)
+ tbl_type = 'TABLE'
+ if table_template.upper().strip().startswith('CREATE VIEW'):
+ tbl_type = 'VIEW'
+ create_stmt += 'DROP %s IF EXISTS %s%s.%s;\n' \
+ % (tbl_type, db_name, db_suffix, table_name)
# hbase / kudu tables are external, and not read from hdfs. We don't need an
# hdfs_location.
if file_format in ['hbase', 'kudu']:
[impala] 01/05: IMPALA-11451: Add net-tools dependency to bootstrap_system.sh
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit b157f1a01a6540f5410376d7bc082523474470d9
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Fri Jul 22 08:23:50 2022 -0700
IMPALA-11451: Add net-tools dependency to bootstrap_system.sh
TestKrpcSocket uses netstat as part of its test. netstat
is provided by the net-tools package on Ubuntu and Centos.
This adds that as a dependency in bootstrap_system.sh.
Docker-based tests had been hitting this test failure,
because they start from a clean docker image.
Testing:
- Ran docker-based tests and TestKrpcSocket now passes
Change-Id: I9ad704e408d4ca4741178d4ea7a857bf30d4cfb6
Reviewed-on: http://gerrit.cloudera.org:8080/18774
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
bin/bootstrap_system.sh | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/bin/bootstrap_system.sh b/bin/bootstrap_system.sh
index 218d2434e..9231d324d 100755
--- a/bin/bootstrap_system.sh
+++ b/bin/bootstrap_system.sh
@@ -219,7 +219,8 @@ ubuntu apt-get --yes install ccache curl gawk g++ gcc apt-utils git ant libffi-d
libkrb5-dev krb5-admin-server krb5-kdc krb5-user libsasl2-dev \
libsasl2-modules libsasl2-modules-gssapi-mit libssl-dev make ninja-build \
python-dev python-setuptools python3-dev python3-setuptools postgresql \
- ssh wget vim-common psmisc lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg
+ ssh wget vim-common psmisc lsof openjdk-8-jdk openjdk-8-source openjdk-8-dbg \
+ net-tools
# Required by Kudu in the minicluster
ubuntu20 apt-get --yes install libtinfo5
ARCH_NAME=$(uname -p)
@@ -266,7 +267,7 @@ redhat sudo yum install -y curl gawk gcc gcc-c++ git krb5-devel krb5-server \
postgresql postgresql-server \
wget vim-common nscd cmake fuse-devel zlib-devel \
psmisc lsof openssh-server redhat-lsb java-1.8.0-openjdk-devel \
- java-1.8.0-openjdk-src python3-devel python3-setuptools
+ java-1.8.0-openjdk-src python3-devel python3-setuptools net-tools
# Enable the Powertools repo for snappy-devel on RedHat 8
redhat8 sudo yum install -y dnf-plugins-core
[impala] 02/05: IMPALA-11446: Push-down NOT_IN predicate to iceberg
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 614dc54dcc14687a11bb64495d4b6d4356ef55fc
Author: LPL <li...@sensorsdata.cn>
AuthorDate: Wed Jul 20 14:31:32 2022 +0800
IMPALA-11446: Push-down NOT_IN predicate to iceberg
Because the column value bounds of the Iceberg meta are not necessarily
a min or max value, NOT_IN cannot be answered using them.
NOT_IN(col, {X, ...}) with (X, Y) doesn't guarantee that X is a value
in col. But it works when the push-down column is the partition column,
it's still very helpful.
Testing:
- add e2e tests
Change-Id: Ib8bdaf6f31a4438e11c4eb27485bb413fe6df9a3
Reviewed-on: http://gerrit.cloudera.org:8080/18760
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/planner/IcebergScanNode.java | 14 +++++++-----
.../QueryTest/iceberg-in-predicate-push-down.test | 26 ++++++++++++++++++++++
2 files changed, 34 insertions(+), 6 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
index 547b02e0d..d065bc723 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanNode.java
@@ -284,11 +284,6 @@ public class IcebergScanNode extends HdfsScanNode {
private UnboundPredicate<Object> convertIcebergPredicate(Analyzer analyzer,
InPredicate predicate) throws ImpalaException {
- // TODO: convert NOT_IN predicate
- if (predicate.isNotIn()) {
- return null;
- }
-
// Do not convert if there is an implicit cast
if (!(predicate.getChild(0) instanceof SlotRef)) {
return null;
@@ -322,7 +317,14 @@ public class IcebergScanNode extends HdfsScanNode {
values.add(value);
}
- return Expressions.in(col.getName(), values);
+ // According to the method:
+ // 'org.apache.iceberg.expressions.InclusiveMetricsEvaluator.MetricsEvalVisitor#notIn'
+ // Expressions.notIn only works when the push-down column is the partition column
+ if (predicate.isNotIn())
+ return Expressions.notIn(col.getName(), values);
+ else {
+ return Expressions.in(col.getName(), values);
+ }
}
private UnboundPredicate<Object> convertIcebergPredicate(IsNullPredicate predicate) {
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-in-predicate-push-down.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-in-predicate-push-down.test
index 91144dd64..319165fb2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-in-predicate-push-down.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-in-predicate-push-down.test
@@ -416,6 +416,32 @@ where
aggregation(SUM, NumRowGroups): 0
====
---- QUERY
+# NOT_IN works when col_i is the partition column
+select
+ count(1)
+from
+ ice_pred_pd1
+where
+ col_i not in (0, 1);
+---- RESULTS
+3
+---- RUNTIME_PROFILE
+aggregation(SUM, NumRowGroups): 1
+====
+---- QUERY
+# NOT_IN does not work because col_dt is not the partition column
+select
+ count(1)
+from
+ ice_pred_pd1
+where
+ col_dt not in (DATE'1400-01-01', DATE'1500-01-01');
+---- RESULTS
+3
+---- RUNTIME_PROFILE
+aggregation(SUM, NumRowGroups): 3
+====
+---- QUERY
# Test DECIMAL(9, 3)/DECIMAL(18, 3)/DECIMAL(38, 3) InPredicate push-down
create table ice_pred_pd2 (
p INT,
[impala] 04/05: IMPALA-11380: Fix trailing whitespace for VerticalOutputFormatter
Posted by st...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit e00a2cbd818dea45a7c6f2fac579af4389d42759
Author: yx91490 <yx...@126.com>
AuthorDate: Wed Jul 13 19:36:15 2022 +0800
IMPALA-11380: Fix trailing whitespace for VerticalOutputFormatter
Similar to IMPALA-11332, The current VerticalOutputFormatter is
stripping trailing whitespaces from the last line of output. This
rstrip() was intended to remove an extra newline,
but it is matching other white space. This is a
problem for a SQL query like:
select 'Trailing whitespace ';
This changes the rstrip() to rstrip('\n') to
avoid removing the other white space.
Testing:
- Current shell tests pass
- Added a shell test that verifies trailing whitespace
is not being stripped.
Change-Id: Id66162d28498e7bef2933651616cf3df2fb0f354
Reviewed-on: http://gerrit.cloudera.org:8080/18722
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
---
shell/shell_output.py | 9 ++++++++-
tests/shell/test_shell_commandline.py | 5 ++++-
2 files changed, 12 insertions(+), 2 deletions(-)
diff --git a/shell/shell_output.py b/shell/shell_output.py
index b5ccbe167..b417ceacc 100644
--- a/shell/shell_output.py
+++ b/shell/shell_output.py
@@ -119,7 +119,14 @@ class VerticalOutputFormatter(DelimitedOutputFormatter):
for c, val in enumerate(row):
row[c] = self.column_names[c].rjust(self.column_name_max_len) + ": " + val
writer.writerow(row)
- rows = temp_buffer.getvalue().rstrip()
+ # The CSV writer produces an extra newline. Strip that extra newline (and
+ # only that extra newline). csv wraps newlines for data values in quotes,
+ # so rstrip will be limited to the extra newline.
+ if sys.version_info.major == 2:
+ # Python 2 is in encoded Unicode bytes, so this needs to be a bytes \n.
+ rows = temp_buffer.getvalue().rstrip(b'\n')
+ else:
+ rows = temp_buffer.getvalue().rstrip('\n')
temp_buffer.close()
return rows
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 3daf42285..dfaeaf814 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -1267,11 +1267,14 @@ class TestImpalaShell(ImpalaTestSuite):
"""Test CSV output with trailing whitespace"""
# Ten trailing spaces
- query = "select 'Trailing Whitespace '"
+ query = "select 'Trailing Whitespace ' as x"
# Only one column, no need for output_delimiter
output = run_impala_shell_cmd(vector, ['-q', query, '-B'])
assert "Fetched 1 row(s)" in output.stderr
assert "Trailing Whitespace \n" in output.stdout
+ output = run_impala_shell_cmd(vector, ['-q', query, '-E'])
+ assert "Fetched 1 row(s)" in output.stderr
+ assert "x: Trailing Whitespace \n" in output.stdout
def test_shell_flush(self, vector, tmp_file):
"""Verify that the rows are flushed before the Fetch X row(s) message"""