You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/02/24 19:13:08 UTC
[1/3] impala git commit: IMPALA-6570: Support KUDU on SLES12.
Repository: impala
Updated Branches:
refs/heads/2.x 1795acaa0 -> 7107f5b84
IMPALA-6570: Support KUDU on SLES12.
Different versions (service packs) of SLES12 report themselves
differently using "lsb_release"; this commit amends impala-config to
identify those multiple variants.
Testing: Built in a SLES12sp2 Docker image.
Change-Id: I3adf17cafda6e3bb1d760f2a8f585bcda540cf9f
Reviewed-on: http://gerrit.cloudera.org:8080/9431
Reviewed-by: David Knupp <dk...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/b0aa94fa
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/b0aa94fa
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/b0aa94fa
Branch: refs/heads/2.x
Commit: b0aa94fa83d69a69d6017a72f3e9202e02dbdcb0
Parents: 18902f0
Author: Philip Zeyliger <ph...@cloudera.com>
Authored: Fri Feb 23 11:33:41 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Feb 24 01:59:35 2018 +0000
----------------------------------------------------------------------
bin/impala-config.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/b0aa94fa/bin/impala-config.sh
----------------------------------------------------------------------
diff --git a/bin/impala-config.sh b/bin/impala-config.sh
index 09476c3..e109573 100755
--- a/bin/impala-config.sh
+++ b/bin/impala-config.sh
@@ -272,7 +272,7 @@ if [[ -z "${KUDU_IS_SUPPORTED-}" ]]; then
# Remove spaces, trim minor versions, and convert to lowercase.
DISTRO_VERSION="$(tr -d ' \n' <<< "$DISTRO_VERSION" | cut -d. -f1 | tr "A-Z" "a-z")"
case "$DISTRO_VERSION" in
- centos6 | centos7 | debian7 | debian8 | suselinux12 | ubuntu* )
+ centos6 | centos7 | debian7 | debian8 | suselinux12 | suse12 | ubuntu* )
KUDU_IS_SUPPORTED=true;;
esac
fi
[2/3] impala git commit: IMPALA-6567: ResetMetadataStmt analysis
should not load tables.
Posted by ta...@apache.org.
IMPALA-6567: ResetMetadataStmt analysis should not load tables.
This fixes a regression introduced by IMPALA-5152 where
invalidate metadata <tbl> and refresh <tbl> accidentally
required the target table to be loaded during analysis,
ultimately leading to a double load in some situations
(load during analysis, then another load during execution).
Since the purpose of these statements is to reload
metadata it does not make sense to require a table load
during analysis - that load happens during execution.
Note that REFRESH <tbl> PARTITION (<partition>) still
requires the containing table to be loaded. This was
the behavior before IMPALA-5152 and this patch does
not attempt to improve that.
Testing:
- added new unit test
- ran FE tests locally
- validated the desired behavior by inspecting logs
and the timeine from invalidate/refresh statements
Change-Id: I7033781ebf27ea53cfd26ff0e4f74d4f242bd1dc
Reviewed-on: http://gerrit.cloudera.org:8080/9418
Tested-by: Impala Public Jenkins
Reviewed-by: Alex Behm <al...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/18902f0e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/18902f0e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/18902f0e
Branch: refs/heads/2.x
Commit: 18902f0ea30d757b6e11795ff8bfbb3aade769cc
Parents: 1795aca
Author: Alex Behm <al...@cloudera.com>
Authored: Thu Feb 22 21:07:27 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Feb 24 01:59:35 2018 +0000
----------------------------------------------------------------------
.../impala/analysis/ResetMetadataStmt.java | 5 ++++-
.../impala/analysis/StmtMetadataLoaderTest.java | 20 ++++++++++++++++++++
2 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/18902f0e/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
index 1fd1e7c..e070d51 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
@@ -81,7 +81,10 @@ public class ResetMetadataStmt extends StatementBase {
@Override
public void collectTableRefs(List<TableRef> tblRefs) {
- if (tableName_ != null) tblRefs.add(new TableRef(tableName_.toPath(), null));
+ // Only need table metadata for REFRESH <tbl> PARTITION (<partition>)
+ if (tableName_ != null && partitionSpec_ != null) {
+ tblRefs.add(new TableRef(tableName_.toPath(), null));
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/impala/blob/18902f0e/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
index f2c8faa..39416f8 100644
--- a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
@@ -47,6 +47,13 @@ public class StmtMetadataLoaderTest {
validateCached(stmt, fe, expectedDbs, expectedTables);
}
+ private void testNoLoad(String stmtStr) throws ImpalaException {
+ ImpaladTestCatalog catalog = new ImpaladTestCatalog();
+ Frontend fe = new Frontend(AuthorizationConfig.createAuthDisabledConfig(), catalog);
+ StatementBase stmt = fe.parse(stmtStr);
+ validateCached(stmt, fe, new String[]{}, new String[]{});
+ }
+
private void validateDbs(StmtTableCache stmtTableCache, String[] expectedDbs) {
String[] actualDbs = new String[stmtTableCache.dbs.size()];
actualDbs = stmtTableCache.dbs.toArray(actualDbs);
@@ -177,4 +184,17 @@ public class StmtMetadataLoaderTest {
new String[] {"functional.view_view", "functional.alltypes_view",
"functional.alltypes"});
}
+
+ @Test
+ public void testResetMetadataStmts() throws ImpalaException {
+ // These stmts should not request any table loads.
+ testNoLoad("invalidate metadata");
+ testNoLoad("invalidate metadata functional.alltypes");
+ testNoLoad("refresh functional.alltypes");
+ testNoLoad("refresh functions functional");
+
+ // This stmt requires the table to be loaded.
+ testLoadTables("refresh functional.alltypes partition (year=2009, month=1)", 1, 1,
+ new String[] {"default", "functional"}, new String[] {"functional.alltypes"});
+ }
}
[3/3] impala git commit: IMPALA-6580: Use LOAD DATA LOCAL for decimal
tables
Posted by ta...@apache.org.
IMPALA-6580: Use LOAD DATA LOCAL for decimal tables
IMPALA-5752 added support for Kudu decimal. As a part
of that, it added Kudu versions of decimal_tbl and
decimal_tiny. Kudu tables are created and loaded even
on local tests, so these tables are loaded when they
previously weren't. The LOAD sections for these tables
rely on executing HDFS commmands to copy data to
appropriate locations. These HDFS commands cannot work
on local tests, causing this failure.
Untangling when to execute LOAD sections is complicated,
so this simply switches the decimal_tbl and decimal_tiny
to do LOAD DATA LOCAL calls, which do not rely on HDFS
commands.
Change-Id: I1f717917269d116c07a6f17944583f5e8faf2932
Reviewed-on: http://gerrit.cloudera.org:8080/9438
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/7107f5b8
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/7107f5b8
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/7107f5b8
Branch: refs/heads/2.x
Commit: 7107f5b840fcf8fff31400fcb4e4e700253ad4a1
Parents: b0aa94f
Author: Joe McDonnell <jo...@cloudera.com>
Authored: Fri Feb 23 16:01:57 2018 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Sat Feb 24 05:42:46 2018 +0000
----------------------------------------------------------------------
testdata/datasets/functional/functional_schema_template.sql | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/7107f5b8/testdata/datasets/functional/functional_schema_template.sql
----------------------------------------------------------------------
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 8db806a..cede525 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -1721,8 +1721,8 @@ ALTER TABLE {table_name} ADD IF NOT EXISTS PARTITION(d6=1);
---- ROW_FORMAT
delimited fields terminated by ','
---- LOAD
-`hadoop fs -mkdir -p /test-warehouse/decimal_tbl/d6=1 && hadoop fs -put -f \
-${IMPALA_HOME}/testdata/data/decimal_tbl.txt /test-warehouse/decimal_tbl/d6=1/
+LOAD DATA LOCAL INPATH '{impala_home}/testdata/data/decimal_tbl.txt'
+OVERWRITE INTO TABLE {db_name}{db_suffix}.{table_name} PARTITION(d6=1);
---- DEPENDENT_LOAD
INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name} partition(d6)
select * from functional.{table_name};
@@ -1755,8 +1755,8 @@ c3 DECIMAL(1,1)
---- ROW_FORMAT
delimited fields terminated by ','
---- LOAD
-`hadoop fs -mkdir -p /test-warehouse/decimal_tiny && hadoop fs -put -f \
-${IMPALA_HOME}/testdata/data/decimal-tiny.txt /test-warehouse/decimal_tiny/
+LOAD DATA LOCAL INPATH '{impala_home}/testdata/data/decimal-tiny.txt'
+OVERWRITE INTO TABLE {db_name}{db_suffix}.{table_name};
---- DEPENDENT_LOAD
INSERT OVERWRITE TABLE {db_name}{db_suffix}.{table_name}
select * from functional.{table_name};