You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by sa...@apache.org on 2016/11/15 18:50:29 UTC
[1/9] incubator-impala git commit: IMPALA-3552: Make incremental
stats max serialized size configurable
Repository: incubator-impala
Updated Branches:
refs/heads/master cac02d6b7 -> b3cbc960a
IMPALA-3552: Make incremental stats max serialized size configurable
The fix "IMPALA-2648/IMPALA-2664" introduced a conservative limitation
on the maximum serialized size of incremental stats. As a side effect,
some users with very large tables are experiencing regressions
especially when they upgrade impala and the serialized size goes
beyond 200MB.
To mitigate the issue, the change introduces a new gflag,
'inc_stats_size_limit_bytes' to make the max serialized size
configurable, which allows impala users to specify their own maximum
serialized size. Default value for inc_stats_size_limit_bytes is
200MB.
The change introduces a TBackendGflags class to pass the gflags from
backend to the Frontend and the Catalog via thrift. This also revamps
existing query options to use the TBackendConfig.
Change-Id: I33684725a61eabc67237503e61178305d37d3cb5
Reviewed-on: http://gerrit.cloudera.org:8080/4867
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/3f2f008a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/3f2f008a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/3f2f008a
Branch: refs/heads/master
Commit: 3f2f008ac4365d7f7c616730d3faeebd776b74d2
Parents: cac02d6
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Fri Nov 4 16:36:39 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 03:22:11 2016 +0000
----------------------------------------------------------------------
be/generated-sources/gen-cpp/CMakeLists.txt | 2 +
be/src/catalog/catalog.cc | 28 ++------
be/src/common/global-flags.cc | 6 ++
be/src/service/fe-support.cc | 18 -----
be/src/service/frontend.cc | 38 +++-------
be/src/util/CMakeLists.txt | 1 +
be/src/util/backend-gflag-util.cc | 76 ++++++++++++++++++++
be/src/util/backend-gflag-util.h | 31 ++++++++
common/thrift/BackendGflags.thrift | 55 ++++++++++++++
common/thrift/CMakeLists.txt | 1 +
common/thrift/Frontend.thrift | 5 --
.../apache/impala/analysis/AlterViewStmt.java | 3 +-
.../impala/analysis/ComputeStatsStmt.java | 10 +--
.../apache/impala/analysis/CreateViewStmt.java | 3 +-
.../org/apache/impala/authorization/User.java | 26 ++-----
.../org/apache/impala/catalog/HdfsTable.java | 11 +--
.../org/apache/impala/catalog/KuduTable.java | 2 +-
.../org/apache/impala/common/RuntimeEnv.java | 12 ----
.../java/org/apache/impala/planner/Planner.java | 3 +-
.../apache/impala/service/BackendConfig.java | 70 ++++++++++--------
.../org/apache/impala/service/FeSupport.java | 18 -----
.../org/apache/impala/service/JniCatalog.java | 30 ++++----
.../org/apache/impala/service/JniFrontend.java | 26 +++----
.../java/org/apache/impala/util/KuduUtil.java | 4 +-
24 files changed, 280 insertions(+), 199 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/generated-sources/gen-cpp/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/generated-sources/gen-cpp/CMakeLists.txt b/be/generated-sources/gen-cpp/CMakeLists.txt
index 35f7d65..ea7735f 100644
--- a/be/generated-sources/gen-cpp/CMakeLists.txt
+++ b/be/generated-sources/gen-cpp/CMakeLists.txt
@@ -39,6 +39,8 @@ set(SRC_FILES
ImpalaHiveServer2Service.cpp
beeswax_constants.cpp
beeswax_types.cpp
+ BackendGflags_constants.cpp
+ BackendGflags_types.cpp
BeeswaxService.cpp
TCLIService.cpp
TCLIService_constants.cpp
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/catalog/catalog.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog.cc b/be/src/catalog/catalog.cc
index 7d6fdca..b3690f6 100644
--- a/be/src/catalog/catalog.cc
+++ b/be/src/catalog/catalog.cc
@@ -22,15 +22,12 @@
#include "common/logging.h"
#include "rpc/jni-thrift-util.h"
-#include "util/logging-support.h"
+#include "util/backend-gflag-util.h"
#include "common/names.h"
using namespace impala;
-DECLARE_bool(load_auth_to_local_rules);
-DECLARE_string(principal);
-DECLARE_string(local_library_dir);
DEFINE_bool(load_catalog_in_background, false,
"If true, loads catalog metadata in the background. If false, metadata is loaded "
@@ -41,13 +38,9 @@ DEFINE_int32(num_metadata_loading_threads, 16,
DEFINE_string(sentry_config, "", "Local path to a sentry-site.xml configuration "
"file. If set, authorization will be enabled.");
-DECLARE_int32(non_impala_java_vlog);
-DECLARE_int32(kudu_operation_timeout_ms);
-
Catalog::Catalog() {
JniMethodDescriptor methods[] = {
- {"<init>", "(ZILjava/lang/String;IIZLjava/lang/String;Ljava/lang/String;I)V",
- &catalog_ctor_},
+ {"<init>", "([B)V", &catalog_ctor_},
{"updateCatalog", "([B)[B", &update_metastore_id_},
{"execDdl", "([B)[B", &exec_ddl_id_},
{"resetMetadata", "([B)[B", &reset_metadata_id_},
@@ -70,19 +63,10 @@ Catalog::Catalog() {
ABORT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, catalog_class_, &(methods[i])));
}
- jboolean load_in_background = FLAGS_load_catalog_in_background;
- jint num_metadata_loading_threads = FLAGS_num_metadata_loading_threads;
- jstring sentry_config = jni_env->NewStringUTF(FLAGS_sentry_config.c_str());
- // auth_to_local rules are read if --load_auth_to_local_rules is set to true
- // and impala is kerberized.
- jboolean auth_to_local = FLAGS_load_auth_to_local_rules && !FLAGS_principal.empty();
- jstring principal = jni_env->NewStringUTF(FLAGS_principal.c_str());
- jstring local_library_dir = jni_env->NewStringUTF(FLAGS_local_library_dir.c_str());
- jint kudu_operation_timeout = FLAGS_kudu_operation_timeout_ms;
- jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_,
- load_in_background, num_metadata_loading_threads, sentry_config,
- FlagToTLogLevel(FLAGS_v), FlagToTLogLevel(FLAGS_non_impala_java_vlog),
- auth_to_local, principal, local_library_dir, kudu_operation_timeout);
+ jbyteArray cfg_bytes;
+ ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+
+ jobject catalog = jni_env->NewObject(catalog_class_, catalog_ctor_, cfg_bytes);
EXIT_IF_EXC(jni_env);
ABORT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, catalog, &catalog_));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/common/global-flags.cc
----------------------------------------------------------------------
diff --git a/be/src/common/global-flags.cc b/be/src/common/global-flags.cc
index f75236c..5a3eff4 100644
--- a/be/src/common/global-flags.cc
+++ b/be/src/common/global-flags.cc
@@ -123,3 +123,9 @@ DEFINE_bool(enable_accept_queue_server, true,
"TThreadedServer that accepts connections as quickly as possible and hands them off "
"to a thread pool to finish setup, reducing the chances that connections time out "
"waiting to be accepted.");
+
+DEFINE_int64(inc_stats_size_limit_bytes, 200 * (1LL<<20), "Maximum size of "
+ "incremental stats the catalog is allowed to serialize per table. "
+ "This limit is set as a safety check, to prevent the JVM from "
+ "hitting a maximum array limit of 1GB (or OOM) while building "
+ "the thrift objects to send to impalads. By default, it's set to 200MB");
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/service/fe-support.cc
----------------------------------------------------------------------
diff --git a/be/src/service/fe-support.cc b/be/src/service/fe-support.cc
index 42ceea2..7147964 100644
--- a/be/src/service/fe-support.cc
+++ b/be/src/service/fe-support.cc
@@ -336,20 +336,6 @@ Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad(
return result_bytes;
}
-extern "C"
-JNIEXPORT jbyteArray JNICALL
-Java_org_apache_impala_service_FeSupport_NativeGetStartupOptions(JNIEnv* env,
- jclass caller_class) {
- TStartupOptions options;
- ExecEnv* exec_env = ExecEnv::GetInstance();
- ImpalaServer* impala_server = exec_env->impala_server();
- options.__set_compute_lineage(impala_server->IsLineageLoggingEnabled());
- jbyteArray result_bytes = NULL;
- THROW_IF_ERROR_RET(SerializeThriftMsg(env, &options, &result_bytes), env,
- JniUtil::internal_exc_class(), result_bytes);
- return result_bytes;
-}
-
namespace impala {
static JNINativeMethod native_methods[] = {
@@ -373,10 +359,6 @@ static JNINativeMethod native_methods[] = {
(char*)"NativePrioritizeLoad", (char*)"([B)[B",
(void*)::Java_org_apache_impala_service_FeSupport_NativePrioritizeLoad
},
- {
- (char*)"NativeGetStartupOptions", (char*)"()[B",
- (void*)::Java_org_apache_impala_service_FeSupport_NativeGetStartupOptions
- },
};
void InitFeSupport() {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/service/frontend.cc
----------------------------------------------------------------------
diff --git a/be/src/service/frontend.cc b/be/src/service/frontend.cc
index f8cecfb..b83d091 100644
--- a/be/src/service/frontend.cc
+++ b/be/src/service/frontend.cc
@@ -22,21 +22,13 @@
#include "common/logging.h"
#include "rpc/jni-thrift-util.h"
+#include "util/backend-gflag-util.h"
#include "util/jni-util.h"
-#include "util/logging-support.h"
#include "common/names.h"
using namespace impala;
-DECLARE_string(sentry_config);
-DECLARE_int32(non_impala_java_vlog);
-DECLARE_bool(load_auth_to_local_rules);
-DECLARE_string(principal);
-DECLARE_int32(kudu_operation_timeout_ms);
-
-DEFINE_bool(load_catalog_at_startup, false, "if true, load all catalog data at startup");
-
// Authorization related flags. Must be set to valid values to properly configure
// authorization.
DEFINE_string(server_name, "", "The name to use for securing this impalad "
@@ -62,10 +54,10 @@ DEFINE_string(authorized_proxy_user_config_delimiter, ",",
DEFINE_string(kudu_master_hosts, "", "Specifies the default Kudu master(s). The given "
"value should be a comma separated list of hostnames or IP addresses; ports are "
"optional.");
+
Frontend::Frontend() {
JniMethodDescriptor methods[] = {
- {"<init>", "(ZLjava/lang/String;Ljava/lang/String;Ljava/lang/String;"
- "Ljava/lang/String;IIZLjava/lang/String;I)V", &fe_ctor_},
+ {"<init>", "([B)V", &fe_ctor_},
{"createExecRequest", "([B)[B", &create_exec_request_id_},
{"getExplainPlan", "([B)Ljava/lang/String;", &get_explain_plan_id_},
{"getHadoopConfig", "([B)[B", &get_hadoop_config_id_},
@@ -89,7 +81,7 @@ Frontend::Frontend() {
{"loadTableData", "([B)[B", &load_table_data_id_},
{"getTableFiles", "([B)[B", &get_table_files_id_},
{"showCreateFunction", "([B)Ljava/lang/String;", &show_create_function_id_},
- {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_},
+ {"buildTestDescriptorTable", "([B)[B", &build_test_descriptor_table_id_}
};
JNIEnv* jni_env = getJNIEnv();
@@ -102,24 +94,10 @@ Frontend::Frontend() {
ABORT_IF_ERROR(JniUtil::LoadJniMethod(jni_env, fe_class_, &(methods[i])));
};
- jboolean lazy = (FLAGS_load_catalog_at_startup ? false : true);
- jstring policy_file_path =
- jni_env->NewStringUTF(FLAGS_authorization_policy_file.c_str());
- jstring server_name =
- jni_env->NewStringUTF(FLAGS_server_name.c_str());
- jstring sentry_config =
- jni_env->NewStringUTF(FLAGS_sentry_config.c_str());
- jstring auth_provider_class =
- jni_env->NewStringUTF(FLAGS_authorization_policy_provider_class.c_str());
- // auth_to_local rules are read if --load_auth_to_local_rules is set to true
- // and impala is kerberized.
- jboolean auth_to_local = FLAGS_load_auth_to_local_rules && !FLAGS_principal.empty();
- jstring kudu_master_hosts = jni_env->NewStringUTF(FLAGS_kudu_master_hosts.c_str());
- jint kudu_operation_timeout = FLAGS_kudu_operation_timeout_ms;
- jobject fe = jni_env->NewObject(fe_class_, fe_ctor_, lazy, server_name,
- policy_file_path, sentry_config, auth_provider_class, FlagToTLogLevel(FLAGS_v),
- FlagToTLogLevel(FLAGS_non_impala_java_vlog), auth_to_local, kudu_master_hosts,
- kudu_operation_timeout);
+ jbyteArray cfg_bytes;
+ ABORT_IF_ERROR(GetThriftBackendGflags(jni_env, &cfg_bytes));
+
+ jobject fe = jni_env->NewObject(fe_class_, fe_ctor_, cfg_bytes);
EXIT_IF_EXC(jni_env);
ABORT_IF_ERROR(JniUtil::LocalToGlobalRef(jni_env, fe, &fe_));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
index ecc222a..6656a38 100644
--- a/be/src/util/CMakeLists.txt
+++ b/be/src/util/CMakeLists.txt
@@ -32,6 +32,7 @@ set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}/util")
add_library(Util
auth-util.cc
avro-util.cc
+ backend-gflag-util.cc
benchmark.cc
bitmap.cc
bit-util.cc
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/backend-gflag-util.cc
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.cc b/be/src/util/backend-gflag-util.cc
new file mode 100644
index 0000000..f504e57
--- /dev/null
+++ b/be/src/util/backend-gflag-util.cc
@@ -0,0 +1,76 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "util/backend-gflag-util.h"
+
+#include "gen-cpp/BackendGflags_types.h"
+#include "rpc/jni-thrift-util.h"
+#include "util/logging-support.h"
+
+#include <gflags/gflags.h>
+
+// Configs for the Frontend and the Catalog.
+DECLARE_bool(load_catalog_in_background);
+DECLARE_bool(load_auth_to_local_rules);
+DECLARE_int32(non_impala_java_vlog);
+DECLARE_int32(read_size);
+DECLARE_int32(num_metadata_loading_threads);
+DECLARE_int32(kudu_operation_timeout_ms);
+DECLARE_int64(inc_stats_size_limit_bytes);
+DECLARE_string(principal);
+DECLARE_string(lineage_event_log_dir);
+DECLARE_string(principal);
+DECLARE_string(local_library_dir);
+DECLARE_string(server_name);
+DECLARE_string(authorization_policy_file);
+DECLARE_string(authorization_policy_provider_class);
+DECLARE_string(authorized_proxy_user_config);
+DECLARE_string(authorized_proxy_user_config_delimiter);
+DECLARE_string(kudu_master_hosts);
+DECLARE_string(sentry_config);
+DECLARE_string(sentry_config);
+
+namespace impala {
+
+Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes) {
+ TBackendGflags cfg;
+ cfg.__set_authorization_policy_file(FLAGS_authorization_policy_file);
+ cfg.__set_load_catalog_in_background(FLAGS_load_catalog_in_background);
+ cfg.__set_server_name(FLAGS_server_name);
+ cfg.__set_sentry_config(FLAGS_sentry_config);
+ cfg.__set_authorization_policy_provider_class(
+ FLAGS_authorization_policy_provider_class);
+ cfg.__set_kudu_master_hosts(FLAGS_kudu_master_hosts);
+ cfg.__set_read_size(FLAGS_read_size);
+ cfg.__set_num_metadata_loading_threads(FLAGS_num_metadata_loading_threads);
+ cfg.__set_sentry_config(FLAGS_sentry_config);
+ // auth_to_local rules are read if --load_auth_to_local_rules is set to true
+ // and impala is kerberized.
+ cfg.__set_load_auth_to_local_rules(FLAGS_load_auth_to_local_rules);
+ cfg.__set_principal(FLAGS_principal);
+ cfg.__set_impala_log_lvl(FlagToTLogLevel(FLAGS_v));
+ cfg.__set_non_impala_java_vlog(FlagToTLogLevel(FLAGS_non_impala_java_vlog));
+ cfg.__set_inc_stats_size_limit_bytes(FLAGS_inc_stats_size_limit_bytes);
+ cfg.__set_lineage_event_log_dir(FLAGS_lineage_event_log_dir);
+ cfg.__set_local_library_path(FLAGS_local_library_dir);
+ cfg.__set_kudu_operation_timeout_ms(FLAGS_kudu_operation_timeout_ms);
+ JniLocalFrame jni_frame;
+ RETURN_IF_ERROR(jni_frame.push(jni_env));
+ RETURN_IF_ERROR(SerializeThriftMsg(jni_env, &cfg, cfg_bytes));
+ return Status::OK();
+}
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/be/src/util/backend-gflag-util.h
----------------------------------------------------------------------
diff --git a/be/src/util/backend-gflag-util.h b/be/src/util/backend-gflag-util.h
new file mode 100644
index 0000000..796a2d5
--- /dev/null
+++ b/be/src/util/backend-gflag-util.h
@@ -0,0 +1,31 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#ifndef UTIL_BACKEND_CONFIG_H_
+#define UTIL_BACKEND_CONFIG_H_
+
+#include "common/status.h"
+#include "util/jni-util.h"
+
+namespace impala {
+
+/// Builds the TBackendGflags object to pass to JNI. This is used to pass the gflag
+/// configs to the Frontend and the Catalog.
+Status GetThriftBackendGflags(JNIEnv* jni_env, jbyteArray* cfg_bytes);
+}
+
+#endif
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/BackendGflags.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/BackendGflags.thrift b/common/thrift/BackendGflags.thrift
new file mode 100644
index 0000000..ad081a6
--- /dev/null
+++ b/common/thrift/BackendGflags.thrift
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+namespace cpp impala
+namespace java org.apache.impala.thrift
+
+// Used to pass gflags from backend to frontend, JniCatalog and JniFrontend
+// Attributes without comments correspond to gflags
+struct TBackendGflags {
+ 1: required string sentry_config
+
+ 2: required bool load_auth_to_local_rules
+
+ 3: required i32 non_impala_java_vlog
+
+ 4: required i32 impala_log_lvl
+
+ 5: required i64 inc_stats_size_limit_bytes
+
+ 6: required string lineage_event_log_dir
+
+ 7: required bool load_catalog_in_background
+
+ 8: required i32 num_metadata_loading_threads
+
+ 9: required string principal
+
+ 10: required string authorization_policy_file
+
+ 11: required string server_name
+
+ 12: required string authorization_policy_provider_class
+
+ 13: required string kudu_master_hosts
+
+ 14: required string local_library_path
+
+ 15: required i32 read_size
+
+ 16: required i32 kudu_operation_timeout_ms
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/common/thrift/CMakeLists.txt b/common/thrift/CMakeLists.txt
index 10d363d..1512e52 100644
--- a/common/thrift/CMakeLists.txt
+++ b/common/thrift/CMakeLists.txt
@@ -144,6 +144,7 @@ set (EXT_DATA_SRC_FILES
set (SRC_FILES
ErrorCodes.thrift
beeswax.thrift
+ BackendGflags.thrift
CatalogInternalService.thrift
CatalogObjects.thrift
CatalogService.thrift
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index f6ba35a..8c279f5 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -721,11 +721,6 @@ struct TGetAllHadoopConfigsResponse {
1: optional map<string, string> configs;
}
-// BE startup options
-struct TStartupOptions {
- 1: optional bool compute_lineage
-}
-
// For creating a test descriptor table. The tuples and their memory layout are computed
// in the FE.
struct TBuildTestDescriptorTableParams {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
index e7d3f66..6740f14 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterViewStmt.java
@@ -22,6 +22,7 @@ import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.RuntimeEnv;
+import org.apache.impala.service.BackendConfig;
import com.google.common.base.Preconditions;
@@ -52,7 +53,7 @@ public class AlterViewStmt extends CreateOrAlterViewStmtBase {
}
createColumnAndViewDefs(analyzer);
- if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+ if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
computeLineageGraph(analyzer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 467faca..5ad3cad 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -33,6 +33,7 @@ import org.apache.impala.catalog.Type;
import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.PrintUtils;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TComputeStatsParams;
import org.apache.impala.thrift.TPartitionStats;
import org.apache.impala.thrift.TTableName;
@@ -284,16 +285,17 @@ public class ComputeStatsStmt extends StatementBase {
}
}
// For incremental stats, estimate the size of intermediate stats and report an
- // error if the estimate is greater than MAX_INCREMENTAL_STATS_SIZE_BYTES.
+ // error if the estimate is greater than --inc_stats_size_limit_bytes in bytes
if (isIncremental_) {
+ long incStatMaxSize = BackendConfig.INSTANCE.getIncStatsMaxSize();
long statsSizeEstimate = hdfsTable.getColumns().size() *
hdfsTable.getPartitions().size() * HdfsTable.STATS_SIZE_PER_COLUMN_BYTES;
- if (statsSizeEstimate > HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES) {
+ if (statsSizeEstimate > incStatMaxSize) {
LOG.error("Incremental stats size estimate for table " + hdfsTable.getName() +
- " exceeded " + HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES + ", estimate = "
+ " exceeded " + incStatMaxSize + ", estimate = "
+ statsSizeEstimate);
throw new AnalysisException("Incremental stats size estimate exceeds "
- + PrintUtils.printBytes(HdfsTable.MAX_INCREMENTAL_STATS_SIZE_BYTES)
+ + PrintUtils.printBytes(incStatMaxSize)
+ ". Please try COMPUTE STATS instead.");
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
index 6e30aeb..6b90083 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateViewStmt.java
@@ -24,6 +24,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.thrift.TAccessEvent;
import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.service.BackendConfig;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -60,7 +61,7 @@ public class CreateViewStmt extends CreateOrAlterViewStmtBase {
TCatalogObjectType.VIEW, Privilege.CREATE.toString()));
createColumnAndViewDefs(analyzer);
- if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+ if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
computeLineageGraph(analyzer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/authorization/User.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/authorization/User.java b/fe/src/main/java/org/apache/impala/authorization/User.java
index 190419b..e29b19f 100644
--- a/fe/src/main/java/org/apache/impala/authorization/User.java
+++ b/fe/src/main/java/org/apache/impala/authorization/User.java
@@ -19,42 +19,24 @@ package org.apache.impala.authorization;
import com.google.common.base.Preconditions;
import com.google.common.annotations.VisibleForTesting;
+
+import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.service.BackendConfig;
import java.io.IOException;
-import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import org.apache.hadoop.security.authentication.util.KerberosName;
/*
* Class that represents a User of an Impala session.
*/
public class User {
- static {
- // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
- // from core-site.xml and use it for principal to short name conversion. If it is not,
- // we use the defaultRule ("RULE:[1:$1] RULE:[2:$1]"), which just extracts the user
- // name from any principal of form a@REALM or a/b@REALM. If auth_to_local is enabled
- // and hadoop.security.auth_to_local is not specified in the hadoop configs, we use
- // the "DEFAULT" rule that just extracts the username from any principal in the
- // cluster's local realm. For more details on principal to short name translation,
- // refer to org.apache.hadoop.security.KerberosName.
- final String defaultRule = "RULE:[1:$1] RULE:[2:$1]";
- final Configuration conf = new Configuration();
- if (BackendConfig.isAuthToLocalEnabled()) {
- KerberosName.setRules(conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
- } else {
- // just extract the simple user name
- KerberosName.setRules(defaultRule);
- }
- }
-
private final String name_;
+ // Refer to BackendConfig.initAuthToLocal() for initialization
+ // of static auth_to_local configs in KerberosName class.
private KerberosName kerberosName_;
public User(String name) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 97831d7..69b4b4e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -63,6 +63,7 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.Pair;
import org.apache.impala.common.PrintUtils;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.ImpalaInternalServiceConstants;
import org.apache.impala.thrift.TAccessLevel;
import org.apache.impala.thrift.TCatalogObjectType;
@@ -169,12 +170,6 @@ public class HdfsTable extends Table {
// Store all the partition ids of an HdfsTable.
private HashSet<Long> partitionIds_ = Sets.newHashSet();
- // Maximum size (in bytes) of incremental stats the catalog is allowed to serialize per
- // table. This limit is set as a safety check, to prevent the JVM from hitting a
- // maximum array limit of 1GB (or OOM) while building the thrift objects to send to
- // impalads.
- public static final long MAX_INCREMENTAL_STATS_SIZE_BYTES = 200 * 1024 * 1024;
-
// Estimate (in bytes) of the incremental stats size per column per partition
public static final long STATS_SIZE_PER_COLUMN_BYTES = 400;
@@ -1593,7 +1588,7 @@ public class HdfsTable extends Table {
* partitions). To prevent the catalog from hitting an OOM error while trying to
* serialize large partition incremental stats, we estimate the stats size and filter
* the incremental stats data from partition objects if the estimate exceeds
- * MAX_INCREMENTAL_STATS_SIZE_BYTES.
+ * --inc_stats_size_limit_bytes
*/
private THdfsTable getTHdfsTable(boolean includeFileDesc, Set<Long> refPartitions) {
// includeFileDesc implies all partitions should be included (refPartitions == null).
@@ -1603,7 +1598,7 @@ public class HdfsTable extends Table {
long statsSizeEstimate =
numPartitions * getColumns().size() * STATS_SIZE_PER_COLUMN_BYTES;
boolean includeIncrementalStats =
- (statsSizeEstimate < MAX_INCREMENTAL_STATS_SIZE_BYTES);
+ (statsSizeEstimate < BackendConfig.INSTANCE.getIncStatsMaxSize());
Map<Long, THdfsPartition> idToPartition = Maps.newHashMap();
for (HdfsPartition partition: partitionMap_.values()) {
long id = partition.getId();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
index 6c30077..7b906a7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/KuduTable.java
@@ -340,7 +340,7 @@ public class KuduTable extends Table {
try (KuduClient client = KuduUtil.createKuduClient(getKuduMasterHosts())) {
org.apache.kudu.client.KuduTable kuduTable = client.openTable(kuduTableName_);
List<LocatedTablet> tablets =
- kuduTable.getTabletsLocations(BackendConfig.getKuduClientTimeoutMs());
+ kuduTable.getTabletsLocations(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
for (LocatedTablet tab: tablets) {
TResultRowBuilder builder = new TResultRowBuilder();
builder.add("-1"); // The Kudu client API doesn't expose tablet row counts.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
index af58b16..d33455a 100644
--- a/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
+++ b/fe/src/main/java/org/apache/impala/common/RuntimeEnv.java
@@ -21,7 +21,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.impala.service.FeSupport;
-import org.apache.impala.thrift.TStartupOptions;
/**
* Contains runtime-specific parameters such as the number of CPU cores. Currently only
@@ -35,21 +34,11 @@ public class RuntimeEnv {
private int numCores_;
- // Indicates if column lineage information should be computed for each query.
- private boolean computeLineage_;
-
// Indicates whether this is an environment for testing.
private boolean isTestEnv_;
public RuntimeEnv() {
reset();
- try {
- TStartupOptions opts = FeSupport.GetStartupOptions();
- computeLineage_ = opts.compute_lineage;
- } catch (InternalException e) {
- LOG.error("Error retrieving BE startup options. Shutting down JVM");
- System.exit(1);
- }
}
/**
@@ -63,7 +52,6 @@ public class RuntimeEnv {
public void setNumCores(int numCores) { this.numCores_ = numCores; }
public void setTestEnv(boolean v) { isTestEnv_ = v; }
public boolean isTestEnv() { return isTestEnv_; }
- public boolean computeLineage() { return computeLineage_; }
public boolean isKuduSupported() {
return "true".equals(System.getenv("KUDU_IS_SUPPORTED"));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/planner/Planner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/Planner.java b/fe/src/main/java/org/apache/impala/planner/Planner.java
index ec4cf7e..52524fb 100644
--- a/fe/src/main/java/org/apache/impala/planner/Planner.java
+++ b/fe/src/main/java/org/apache/impala/planner/Planner.java
@@ -39,6 +39,7 @@ import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.common.RuntimeEnv;
import org.apache.impala.common.TreeNode;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TExplainLevel;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TQueryExecRequest;
@@ -179,7 +180,7 @@ public class Planner {
ctx_.getRootAnalyzer().getTimeline().markEvent("Distributed plan created");
ColumnLineageGraph graph = ctx_.getRootAnalyzer().getColumnLineageGraph();
- if (RuntimeEnv.INSTANCE.computeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
+ if (BackendConfig.INSTANCE.getComputeLineage() || RuntimeEnv.INSTANCE.isTestEnv()) {
// Compute the column lineage graph
if (ctx_.isInsertOrCtas()) {
Table targetTable = ctx_.getAnalysisResult().getInsertStmt().getTargetTable();
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/BackendConfig.java b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
index c90f674..47c987e 100644
--- a/fe/src/main/java/org/apache/impala/service/BackendConfig.java
+++ b/fe/src/main/java/org/apache/impala/service/BackendConfig.java
@@ -18,46 +18,60 @@
package org.apache.impala.service;
import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+
+import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
+import org.apache.hadoop.security.authentication.util.KerberosName;
+import org.apache.impala.thrift.TBackendGflags;
/**
* This class is meant to provide the FE with impalad backend configuration parameters,
* including command line arguments.
- * TODO: Remove this class and either
- * a) Figure out if there's a standard way to access flags from java
- * b) Create a util/gflags.java that let's us access the be flags
*/
public class BackendConfig {
- public static BackendConfig INSTANCE = new BackendConfig();
-
- // Default read block size (in bytes). This is the same as
- // the default FLAGS_read_size used by the IO manager in the backend.
- private final long READ_SIZE;
-
- // Variables below are overriden by JniFrontend/JniCatalog with user set configuration.
- // TODO: Read from backend instead of using static variables.
-
- // Determines how principal to short name conversion works. See User.java for more info.
- private static boolean allowAuthToLocalRules_ = false;
+ public static BackendConfig INSTANCE;
- // Kudu client timeout (ms).
- private static int kuduOperationTimeoutMs_ = 3 * 60 * 1000;
+ private TBackendGflags backendCfg_;
- private BackendConfig() {
- // TODO: Populate these by making calls to the backend instead of default constants.
- READ_SIZE = 8 * 1024 * 1024L;
+ private BackendConfig(TBackendGflags cfg) {
+ backendCfg_ = cfg;
}
- public long getReadSize() { return READ_SIZE; }
-
- public static boolean isAuthToLocalEnabled() { return allowAuthToLocalRules_; }
- public static void setAuthToLocal(boolean authToLocal) {
- allowAuthToLocalRules_ = authToLocal;
+ public static void create(TBackendGflags cfg) {
+ Preconditions.checkNotNull(cfg);
+ INSTANCE = new BackendConfig(cfg);
+ initAuthToLocal();
}
- public static int getKuduClientTimeoutMs() { return kuduOperationTimeoutMs_; }
+ public long getReadSize() { return backendCfg_.read_size; }
+ public boolean getComputeLineage() {
+ return !Strings.isNullOrEmpty(backendCfg_.lineage_event_log_dir);
+ }
+ public long getIncStatsMaxSize() { return backendCfg_.inc_stats_size_limit_bytes; }
+ public boolean isAuthToLocalEnabled() {
+ return backendCfg_.load_auth_to_local_rules &&
+ !Strings.isNullOrEmpty(backendCfg_.principal);
+ }
+ public int getKuduClientTimeoutMs() { return backendCfg_.kudu_operation_timeout_ms; }
- public static void setKuduClientTimeoutMs(int kuduOperationTimeoutMs) {
- Preconditions.checkArgument(kuduOperationTimeoutMs > 0);
- BackendConfig.kuduOperationTimeoutMs_ = kuduOperationTimeoutMs;
+ // Inits the auth_to_local configuration in the static KerberosName class.
+ private static void initAuthToLocal() {
+ // If auth_to_local is enabled, we read the configuration hadoop.security.auth_to_local
+ // from core-site.xml and use it for principal to short name conversion. If it is not,
+ // we use the defaultRule ("RULE:[1:$1] RULE:[2:$1]"), which just extracts the user
+ // name from any principal of form a@REALM or a/b@REALM. If auth_to_local is enabled
+ // and hadoop.security.auth_to_local is not specified in the hadoop configs, we use
+ // the "DEFAULT" rule that just extracts the username from any principal in the
+ // cluster's local realm. For more details on principal to short name translation,
+ // refer to org.apache.hadoop.security.KerberosName.
+ final String defaultRule = "RULE:[1:$1] RULE:[2:$1]";
+ final Configuration conf = new Configuration();
+ if (INSTANCE.isAuthToLocalEnabled()) {
+ KerberosName.setRules(conf.get(HADOOP_SECURITY_AUTH_TO_LOCAL, "DEFAULT"));
+ } else {
+ // just extract the simple user name
+ KerberosName.setRules(defaultRule);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/FeSupport.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/FeSupport.java b/fe/src/main/java/org/apache/impala/service/FeSupport.java
index f982df3..c578726 100644
--- a/fe/src/main/java/org/apache/impala/service/FeSupport.java
+++ b/fe/src/main/java/org/apache/impala/service/FeSupport.java
@@ -45,7 +45,6 @@ import org.apache.impala.thrift.TPrioritizeLoadResponse;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TStatus;
-import org.apache.impala.thrift.TStartupOptions;
import org.apache.impala.thrift.TSymbolLookupParams;
import org.apache.impala.thrift.TSymbolLookupResult;
import org.apache.impala.thrift.TTable;
@@ -84,9 +83,6 @@ public class FeSupport {
// using Java Thrift bindings.
public native static byte[] NativePrioritizeLoad(byte[] thriftReq);
- // Return select BE startup options as a serialized TStartupOptions
- public native static byte[] NativeGetStartupOptions();
-
/**
* Locally caches the jar at the specified HDFS location.
*
@@ -263,20 +259,6 @@ public class FeSupport {
}
}
- public static TStartupOptions GetStartupOptions() throws InternalException {
- try {
- byte[] result = NativeGetStartupOptions();
- Preconditions.checkNotNull(result);
- TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
- TStartupOptions options = new TStartupOptions();
- deserializer.deserialize(options, result);
- return options;
- } catch (TException e) {
- throw new InternalException("Error retrieving startup options: " + e.getMessage(),
- e);
- }
- }
-
/**
* This function should only be called explicitly by the FeSupport to ensure that
* native functions are loaded.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniCatalog.java b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
index 7993191..db9ba63 100644
--- a/fe/src/main/java/org/apache/impala/service/JniCatalog.java
+++ b/fe/src/main/java/org/apache/impala/service/JniCatalog.java
@@ -47,6 +47,7 @@ import org.apache.impala.thrift.TResetMetadataRequest;
import org.apache.impala.thrift.TSentryAdminCheckRequest;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogRequest;
+import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.util.GlogAppender;
import org.apache.impala.util.PatternMatcher;
import org.apache.thrift.TException;
@@ -78,29 +79,30 @@ public class JniCatalog {
return new TUniqueId(uuid.getMostSignificantBits(), uuid.getLeastSignificantBits());
}
- public JniCatalog(boolean loadInBackground, int numMetadataLoadingThreads,
- String sentryServiceConfig, int impalaLogLevel, int otherLogLevel,
- boolean allowAuthToLocal, String kerberosPrincipal, String localLibraryPath,
- int kuduOperationTimeoutMs) throws InternalException {
- BackendConfig.setAuthToLocal(allowAuthToLocal);
- BackendConfig.setKuduClientTimeoutMs(kuduOperationTimeoutMs);
- Preconditions.checkArgument(numMetadataLoadingThreads > 0);
+ public JniCatalog(byte[] thriftBackendConfig) throws InternalException,
+ ImpalaException, TException {
+ TBackendGflags cfg = new TBackendGflags();
+ JniUtil.deserializeThrift(protocolFactory_, cfg, thriftBackendConfig);
+
+ BackendConfig.create(cfg);
+
+ Preconditions.checkArgument(cfg.num_metadata_loading_threads > 0);
// This trick saves having to pass a TLogLevel enum, which is an object and more
// complex to pass through JNI.
- GlogAppender.Install(TLogLevel.values()[impalaLogLevel],
- TLogLevel.values()[otherLogLevel]);
+ GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
+ TLogLevel.values()[cfg.non_impala_java_vlog]);
// Check if the Sentry Service is configured. If so, create a configuration object.
SentryConfig sentryConfig = null;
- if (!Strings.isNullOrEmpty(sentryServiceConfig)) {
- sentryConfig = new SentryConfig(sentryServiceConfig);
+ if (!Strings.isNullOrEmpty(cfg.sentry_config)) {
+ sentryConfig = new SentryConfig(cfg.sentry_config);
sentryConfig.loadConfig();
}
LOG.info(JniUtil.getJavaVersion());
- catalog_ = new CatalogServiceCatalog(loadInBackground,
- numMetadataLoadingThreads, sentryConfig, getServiceId(), kerberosPrincipal,
- localLibraryPath);
+ catalog_ = new CatalogServiceCatalog(cfg.load_catalog_in_background,
+ cfg.num_metadata_loading_threads, sentryConfig, getServiceId(), cfg.principal,
+ cfg.local_library_path);
try {
catalog_.reset();
} catch (CatalogException e) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index 3ef89e3..5e41af4 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -46,6 +46,7 @@ import org.apache.impala.common.FileSystemUtil;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.common.JniUtil;
+import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TBuildTestDescriptorTableParams;
import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TDatabase;
@@ -81,6 +82,7 @@ import org.apache.impala.thrift.TTableName;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
import org.apache.impala.thrift.TUpdateMembershipRequest;
+import org.apache.impala.thrift.TBackendGflags;
import org.apache.impala.util.GlogAppender;
import org.apache.impala.util.PatternMatcher;
import org.apache.impala.util.TSessionStateUtil;
@@ -115,20 +117,20 @@ public class JniFrontend {
/**
* Create a new instance of the Jni Frontend.
*/
- public JniFrontend(boolean lazy, String serverName, String authorizationPolicyFile,
- String sentryConfigFile, String authPolicyProviderClass, int impalaLogLevel,
- int otherLogLevel, boolean allowAuthToLocal, String defaultKuduMasterHosts,
- int kuduOperationTimeoutMs)
- throws InternalException {
- BackendConfig.setAuthToLocal(allowAuthToLocal);
- BackendConfig.setKuduClientTimeoutMs(kuduOperationTimeoutMs);
- GlogAppender.Install(TLogLevel.values()[impalaLogLevel],
- TLogLevel.values()[otherLogLevel]);
+ public JniFrontend(byte[] thriftBackendConfig) throws ImpalaException, TException {
+ TBackendGflags cfg = new TBackendGflags();
+ JniUtil.deserializeThrift(protocolFactory_, cfg, thriftBackendConfig);
+
+ BackendConfig.create(cfg);
+
+ GlogAppender.Install(TLogLevel.values()[cfg.impala_log_lvl],
+ TLogLevel.values()[cfg.non_impala_java_vlog]);
// Validate the authorization configuration before initializing the Frontend.
// If there are any configuration problems Impala startup will fail.
- AuthorizationConfig authConfig = new AuthorizationConfig(serverName,
- authorizationPolicyFile, sentryConfigFile, authPolicyProviderClass);
+ AuthorizationConfig authConfig = new AuthorizationConfig(cfg.server_name,
+ cfg.authorization_policy_file, cfg.sentry_config,
+ cfg.authorization_policy_provider_class);
authConfig.validateConfig();
if (authConfig.isEnabled()) {
LOG.info(String.format("Authorization is 'ENABLED' using %s",
@@ -139,7 +141,7 @@ public class JniFrontend {
}
LOG.info(JniUtil.getJavaVersion());
- frontend_ = new Frontend(authConfig, defaultKuduMasterHosts);
+ frontend_ = new Frontend(authConfig, cfg.kudu_master_hosts);
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/3f2f008a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/KuduUtil.java b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
index c50bf25..65fae74 100644
--- a/fe/src/main/java/org/apache/impala/util/KuduUtil.java
+++ b/fe/src/main/java/org/apache/impala/util/KuduUtil.java
@@ -54,8 +54,8 @@ public class KuduUtil {
*/
public static KuduClient createKuduClient(String kuduMasters) {
KuduClientBuilder b = new KuduClient.KuduClientBuilder(kuduMasters);
- b.defaultAdminOperationTimeoutMs(BackendConfig.getKuduClientTimeoutMs());
- b.defaultOperationTimeoutMs(BackendConfig.getKuduClientTimeoutMs());
+ b.defaultAdminOperationTimeoutMs(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
+ b.defaultOperationTimeoutMs(BackendConfig.INSTANCE.getKuduClientTimeoutMs());
return b.build();
}
[3/9] incubator-impala git commit: IMPALA-1654: General partition
exprs in DDL operations.
Posted by sa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
index 8dab33a..077c4cb 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TupleDescriptor.java
@@ -345,4 +345,18 @@ public class TupleDescriptor {
}
return true;
}
+
+ /**
+ * Returns a list of slot ids that correspond to partition columns.
+ */
+ public List<SlotId> getPartitionSlots() {
+ List<SlotId> partitionSlots = Lists.newArrayList();
+ for (SlotDescriptor slotDesc: getSlots()) {
+ if (slotDesc.getColumn() == null) continue;
+ if (slotDesc.getColumn().getPosition() < getTable().getNumClusteringCols()) {
+ partitionSlots.add(slotDesc.getId());
+ }
+ }
+ return partitionSlots;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index f56f502..203ef6d 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -932,6 +932,25 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
+ * Drops the partitions specified in 'partitionSet' from 'tbl'. Throws a
+ * CatalogException if 'tbl' is not an HdfsTable. Returns the target table.
+ */
+ public Table dropPartitions(Table tbl, List<List<TPartitionKeyValue>> partitionSet)
+ throws CatalogException {
+ Preconditions.checkNotNull(tbl);
+ Preconditions.checkNotNull(partitionSet);
+ Preconditions.checkState(Thread.holdsLock(tbl));
+ if (!(tbl instanceof HdfsTable)) {
+ throw new CatalogException("Table " + tbl.getFullName() + " is not an Hdfs table");
+ }
+ HdfsTable hdfsTable = (HdfsTable) tbl;
+ List<HdfsPartition> partitions =
+ hdfsTable.getPartitionsFromPartitionSet(partitionSet);
+ hdfsTable.dropPartitions(partitions);
+ return hdfsTable;
+ }
+
+ /**
* Drops the partition specified in 'partitionSpec' from 'tbl'. Throws a
* CatalogException if 'tbl' is not an HdfsTable. If the partition having the given
* partition spec does not exist, null is returned. Otherwise, the modified table is
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index 69b4b4e..21b5359 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -90,12 +90,42 @@ import org.apache.impala.util.MetaStoreUtil;
import org.apache.impala.util.TAccessLevelUtil;
import org.apache.impala.util.TResultRowBuilder;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableMap;
+
+import org.apache.avro.Schema;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.BlockLocation;
+import org.apache.hadoop.fs.BlockStorageLocation;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.VolumeId;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hive.metastore.IMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.Partition;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.util.StringUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
/**
* Internal representation of table-related metadata of a file-resident table on a
@@ -621,6 +651,20 @@ public class HdfsTable extends Table {
}
/**
+ * Gets hdfs partitions by the given partition set.
+ */
+ public List<HdfsPartition> getPartitionsFromPartitionSet(
+ List<List<TPartitionKeyValue>> partitionSet) {
+ List<HdfsPartition> partitions = Lists.newArrayList();
+ for (List<TPartitionKeyValue> kv : partitionSet) {
+ HdfsPartition partition =
+ getPartitionFromThriftPartitionSpec(kv);
+ if (partition != null) partitions.add(partition);
+ }
+ return partitions;
+ }
+
+ /**
* Create columns corresponding to fieldSchemas. Throws a TableLoadingException if the
* metadata is incompatible with what we support.
*/
@@ -961,6 +1005,20 @@ public class HdfsTable extends Table {
}
/**
+ * Drops the given partitions from this table. Cleans up its metadata from all the
+ * mappings used to speed up partition pruning/lookup. Also updates partitions column
+ * statistics. Returns the list of partitions that were dropped.
+ */
+ public List<HdfsPartition> dropPartitions(List<HdfsPartition> partitions) {
+ ArrayList<HdfsPartition> droppedPartitions = Lists.newArrayList();
+ for (HdfsPartition partition: partitions) {
+ HdfsPartition hdfsPartition = dropPartition(partition);
+ if (hdfsPartition != null) droppedPartitions.add(hdfsPartition);
+ }
+ return droppedPartitions;
+ }
+
+ /**
* Adds or replaces the default partition.
*/
public void addDefaultPartition(StorageDescriptor storageDescriptor)
@@ -1143,7 +1201,7 @@ public class HdfsTable extends Table {
partitionNames.add(partition.getPartitionName());
}
partitionsToRemove.addAll(dirtyPartitions);
- for (HdfsPartition partition: partitionsToRemove) dropPartition(partition);
+ dropPartitions(partitionsToRemove);
// Load dirty partitions from Hive Metastore
loadPartitionsFromMetastore(dirtyPartitions, client);
@@ -1889,7 +1947,7 @@ public class HdfsTable extends Table {
* Returns files info for all partitions, if partition spec is null, ordered
* by partition.
*/
- public TResultSet getFiles(List<TPartitionKeyValue> partitionSpec)
+ public TResultSet getFiles(List<List<TPartitionKeyValue>> partitionSet)
throws CatalogException {
TResultSet result = new TResultSet();
TResultSetMetadata resultSchema = new TResultSetMetadata();
@@ -1899,16 +1957,14 @@ public class HdfsTable extends Table {
resultSchema.addToColumns(new TColumn("Partition", Type.STRING.toThrift()));
result.setRows(Lists.<TResultRow>newArrayList());
- List<HdfsPartition> orderedPartitions = null;
- if (partitionSpec == null) {
+ List<HdfsPartition> orderedPartitions;
+ if (partitionSet == null) {
orderedPartitions = Lists.newArrayList(partitionMap_.values());
- Collections.sort(orderedPartitions);
} else {
- // Get the HdfsPartition object for the given partition spec.
- HdfsPartition partition = getPartitionFromThriftPartitionSpec(partitionSpec);
- Preconditions.checkState(partition != null);
- orderedPartitions = Lists.newArrayList(partition);
+ // Get a list of HdfsPartition objects for the given partition set.
+ orderedPartitions = getPartitionsFromPartitionSet(partitionSet);
}
+ Collections.sort(orderedPartitions);
for (HdfsPartition p: orderedPartitions) {
List<FileDescriptor> orderedFds = Lists.newArrayList(p.getFileDescriptors());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/catalog/Table.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Table.java b/fe/src/main/java/org/apache/impala/catalog/Table.java
index bef381a..be9dc7b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Table.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Table.java
@@ -245,6 +245,10 @@ public abstract class Table implements CatalogObject {
return newTable;
}
+ public boolean isClusteringColumn(Column c) {
+ return c.getPosition() < numClusteringCols_;
+ }
+
protected void loadFromThrift(TTable thriftTable) throws TableLoadingException {
List<TColumn> columns = new ArrayList<TColumn>();
columns.addAll(thriftTable.getClustering_columns());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
index 10b6539..f719d0c 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsPartitionPruner.java
@@ -74,8 +74,7 @@ public class HdfsPartitionPruner {
private final static int PARTITION_PRUNING_BATCH_SIZE = 1024;
private final HdfsTable tbl_;
-
- private final List<SlotId> partitionSlots_ = Lists.newArrayList();
+ private final List<SlotId> partitionSlots_;
// For converting BetweenPredicates to CompoundPredicates so they can be
// executed in the BE.
@@ -85,21 +84,17 @@ public class HdfsPartitionPruner {
public HdfsPartitionPruner(TupleDescriptor tupleDesc) {
Preconditions.checkState(tupleDesc.getTable() instanceof HdfsTable);
tbl_ = (HdfsTable)tupleDesc.getTable();
+ partitionSlots_ = tupleDesc.getPartitionSlots();
- // Collect all the partitioning columns from TupleDescriptor.
- for (SlotDescriptor slotDesc: tupleDesc.getSlots()) {
- if (slotDesc.getColumn() == null) continue;
- if (slotDesc.getColumn().getPosition() < tbl_.getNumClusteringCols()) {
- partitionSlots_.add(slotDesc.getId());
- }
- }
}
/**
* Return a list of partitions left after applying the conjuncts. Please note
- * that conjuncts used for filtering will be removed from the list 'conjuncts'.
+ * that conjunts used for filtering will be removed from the list 'conjuncts'.
+ * If 'allowEmpty' is False, empty partitions are not returned.
*/
- public List<HdfsPartition> prunePartitions(Analyzer analyzer, List<Expr> conjuncts)
+ public List<HdfsPartition> prunePartitions(
+ Analyzer analyzer, List<Expr> conjuncts, boolean allowEmpty)
throws InternalException, AnalysisException {
// Start with creating a collection of partition filters for the applicable conjuncts.
List<HdfsPartitionFilter> partitionFilters = Lists.newArrayList();
@@ -159,7 +154,7 @@ public class HdfsPartitionPruner {
for (Long id: matchingPartitionIds) {
HdfsPartition partition = partitionMap.get(id);
Preconditions.checkNotNull(partition);
- if (partition.hasFileDescriptors()) {
+ if (partition.hasFileDescriptors() || allowEmpty) {
results.add(partition);
analyzer.getDescTbl().addReferencedPartition(tbl_, partition.getId());
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 9b0e376..6219127 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -1196,7 +1196,7 @@ public class SingleNodePlanner {
// Do partition pruning before deciding which slots to materialize,
// We might end up removing some predicates.
HdfsPartitionPruner pruner = new HdfsPartitionPruner(tupleDesc);
- List<HdfsPartition> partitions = pruner.prunePartitions(analyzer, conjuncts);
+ List<HdfsPartition> partitions = pruner.prunePartitions(analyzer, conjuncts, false);
// Mark all slots referenced by the remaining conjuncts as materialized.
analyzer.materializeSlots(conjuncts);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 884c43a..1755934 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -49,6 +49,15 @@ import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
+import org.apache.impala.common.Reference;
+import org.apache.log4j.Logger;
+import org.apache.thrift.TException;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
import org.apache.impala.analysis.FunctionName;
import org.apache.impala.analysis.TableName;
import org.apache.impala.authorization.User;
@@ -332,6 +341,11 @@ public class CatalogOpExecutor {
// When true, loads the table schema and the column stats from the Hive Metastore.
boolean reloadTableSchema = false;
+ // When true, sets the result to be reported to the client.
+ boolean setResultSet = false;
+ TColumnValue resultColVal = new TColumnValue();
+ Reference<Long> numUpdatedPartitions = new Reference<>(0L);
+
TableName tableName = TableName.fromThrift(params.getTable_name());
Table tbl = getExistingTable(tableName.getDb(), tableName.getTbl());
catalog_.getLock().writeLock().lock();
@@ -351,6 +365,7 @@ public class CatalogOpExecutor {
}
// Get a new catalog version to assign to the table being altered.
long newCatalogVersion = catalog_.incrementAndGetCatalogVersion();
+ boolean reloadMetadata = true;
catalog_.getLock().writeLock().unlock();
switch (params.getAlter_type()) {
case ADD_REPLACE_COLUMNS:
@@ -373,7 +388,8 @@ public class CatalogOpExecutor {
refreshedTable.setCatalogVersion(newCatalogVersion);
addTableToCatalogUpdate(refreshedTable, response.result);
}
- return;
+ reloadMetadata = false;
+ break;
case DROP_COLUMN:
TAlterTableDropColParams dropColParams = params.getDrop_col_params();
alterTableDropCol(tbl, dropColParams.getCol_name());
@@ -390,57 +406,82 @@ public class CatalogOpExecutor {
params.getDrop_partition_params();
// Drop the partition from the corresponding table. Get the table object
// with an updated catalog version. If the partition does not exist and
- // "IfExists" is true, then return without populating the response object.
- // If "purge" option is specified partition data is purged by skipping
- // Trash, if configured.
- refreshedTable = alterTableDropPartition(tbl,
- dropPartParams.getPartition_spec(),
- dropPartParams.isIf_exists(), dropPartParams.isPurge());
+ // "IfExists" is true, null is returned. If "purge" option is specified
+ // partition data is purged by skipping Trash, if configured.
+ refreshedTable = alterTableDropPartition(
+ tbl, dropPartParams.getPartition_set(),
+ dropPartParams.isIf_exists(),
+ dropPartParams.isPurge(), numUpdatedPartitions);
if (refreshedTable != null) {
refreshedTable.setCatalogVersion(newCatalogVersion);
addTableToCatalogUpdate(refreshedTable, response.result);
}
- return;
+ resultColVal.setString_val(
+ "Dropped " + numUpdatedPartitions.getRef() + " partition(s).");
+ setResultSet = true;
+ reloadMetadata = false;
+ break;
case RENAME_TABLE:
case RENAME_VIEW:
Preconditions.checkState(false,
"RENAME TABLE/VIEW operation has been processed");
- return;
+ break;
case SET_FILE_FORMAT:
TAlterTableSetFileFormatParams fileFormatParams =
params.getSet_file_format_params();
- List<TPartitionKeyValue> fileFormatPartitionSpec = null;
- if (fileFormatParams.isSetPartition_spec()) {
- fileFormatPartitionSpec = fileFormatParams.getPartition_spec();
+ reloadFileMetadata = alterTableSetFileFormat(
+ tbl, fileFormatParams.getPartition_set(), fileFormatParams.getFile_format(),
+ numUpdatedPartitions);
+
+ if (fileFormatParams.isSetPartition_set()) {
+ resultColVal.setString_val(
+ "Updated " + numUpdatedPartitions.getRef() + " partition(s).");
+ } else {
+ resultColVal.setString_val("Updated table.");
}
- reloadFileMetadata = alterTableSetFileFormat(tbl, fileFormatPartitionSpec,
- fileFormatParams.getFile_format());
+ setResultSet = true;
break;
case SET_LOCATION:
TAlterTableSetLocationParams setLocationParams =
params.getSet_location_params();
- List<TPartitionKeyValue> partitionSpec = null;
- if (setLocationParams.isSetPartition_spec()) {
- partitionSpec = setLocationParams.getPartition_spec();
- }
- reloadFileMetadata = alterTableSetLocation(tbl, partitionSpec,
- setLocationParams.getLocation());
+ reloadFileMetadata = alterTableSetLocation(tbl,
+ setLocationParams.getPartition_spec(), setLocationParams.getLocation());
break;
case SET_TBL_PROPERTIES:
- alterTableSetTblProperties(tbl, params.getSet_tbl_properties_params());
+ alterTableSetTblProperties(tbl, params.getSet_tbl_properties_params(),
+ numUpdatedPartitions);
+ if (params.getSet_tbl_properties_params().isSetPartition_set()) {
+ resultColVal.setString_val(
+ "Updated " + numUpdatedPartitions.getRef() + " partition(s).");
+ } else {
+ resultColVal.setString_val("Updated table.");
+ }
+ setResultSet = true;
break;
case UPDATE_STATS:
Preconditions.checkState(params.isSetUpdate_stats_params());
- alterTableUpdateStats(tbl, params.getUpdate_stats_params(), response);
+ Reference<Long> numUpdatedColumns = new Reference<>(0L);
+ alterTableUpdateStats(tbl, params.getUpdate_stats_params(), response,
+ numUpdatedPartitions, numUpdatedColumns);
reloadTableSchema = true;
+ resultColVal.setString_val("Updated " + numUpdatedPartitions.getRef() +
+ " partition(s) and " + numUpdatedColumns.getRef() + " column(s).");
+ setResultSet = true;
break;
case SET_CACHED:
Preconditions.checkState(params.isSetSet_cached_params());
- if (params.getSet_cached_params().getPartition_spec() == null) {
+ String op = params.getSet_cached_params().getCache_op().isSet_cached() ?
+ "Cached " : "Uncached ";
+ if (params.getSet_cached_params().getPartition_set() == null) {
reloadFileMetadata = alterTableSetCached(tbl, params.getSet_cached_params());
+ resultColVal.setString_val(op + "table.");
} else {
- alterPartitionSetCached(tbl, params.getSet_cached_params());
+ alterPartitionSetCached(tbl, params.getSet_cached_params(),
+ numUpdatedPartitions);
+ resultColVal.setString_val(
+ op + numUpdatedPartitions.getRef() + " partition(s).");
}
+ setResultSet = true;
break;
case RECOVER_PARTITIONS:
alterTableRecoverPartitions(tbl);
@@ -450,9 +491,21 @@ public class CatalogOpExecutor {
"Unknown ALTER TABLE operation type: " + params.getAlter_type());
}
- loadTableMetadata(tbl, newCatalogVersion, reloadFileMetadata, reloadTableSchema,
- null);
- addTableToCatalogUpdate(tbl, response.result);
+ if (reloadMetadata) {
+ loadTableMetadata(tbl, newCatalogVersion, reloadFileMetadata, reloadTableSchema,
+ null);
+ addTableToCatalogUpdate(tbl, response.result);
+ }
+
+ if (setResultSet) {
+ TResultSet resultSet = new TResultSet();
+ resultSet.setSchema(new TResultSetMetadata(Lists.newArrayList(
+ new TColumn("summary", Type.STRING.toThrift()))));
+ TResultRow resultRow = new TResultRow();
+ resultRow.setColVals(Lists.newArrayList(resultColVal));
+ resultSet.setRows(Lists.newArrayList(resultRow));
+ response.setResult_set(resultSet);
+ }
} // end of synchronized block
}
@@ -550,7 +603,8 @@ public class CatalogOpExecutor {
* in batches of size 'MAX_PARTITION_UPDATES_PER_RPC'.
*/
private void alterTableUpdateStats(Table table, TAlterTableUpdateStatsParams params,
- TDdlExecResponse resp) throws ImpalaException {
+ TDdlExecResponse resp, Reference<Long> numUpdatedPartitions,
+ Reference<Long> numUpdatedColumns) throws ImpalaException {
Preconditions.checkState(Thread.holdsLock(table));
if (params.isSetTable_stats()) {
// Updating table and column stats via COMPUTE STATS.
@@ -578,9 +632,9 @@ public class CatalogOpExecutor {
}
}
- int numTargetedPartitions = 0;
- int numUpdatedColumns = 0;
- try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
+ long numTargetedPartitions = 0L;
+ long numTargetedColumns = 0L;
+ try(MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
// Update the table and partition row counts based on the query results.
List<HdfsPartition> modifiedParts = Lists.newArrayList();
if (params.isSetTable_stats()) {
@@ -592,12 +646,12 @@ public class CatalogOpExecutor {
if (params.isSetColumn_stats()) {
// Create Hive column stats from the query results.
colStats = createHiveColStats(params.getColumn_stats(), table);
- numUpdatedColumns = colStats.getStatsObjSize();
+ numTargetedColumns = colStats.getStatsObjSize();
}
// Update all partitions.
bulkAlterPartitions(table.getDb().getName(), table.getName(), modifiedParts);
- if (numUpdatedColumns > 0) {
+ if (numTargetedColumns > 0) {
Preconditions.checkNotNull(colStats);
// Update column stats.
try {
@@ -611,18 +665,8 @@ public class CatalogOpExecutor {
// lastDdlTime is as accurate as possible.
applyAlterTable(msTbl);
}
-
- // Set the results to be reported to the client.
- TResultSet resultSet = new TResultSet();
- resultSet.setSchema(new TResultSetMetadata(Lists.newArrayList(
- new TColumn("summary", Type.STRING.toThrift()))));
- TColumnValue resultColVal = new TColumnValue();
- resultColVal.setString_val("Updated " + numTargetedPartitions + " partition(s) and " +
- numUpdatedColumns + " column(s).");
- TResultRow resultRow = new TResultRow();
- resultRow.setColVals(Lists.newArrayList(resultColVal));
- resultSet.setRows(Lists.newArrayList(resultRow));
- resp.setResult_set(resultSet);
+ numUpdatedPartitions.setRef(numTargetedPartitions);
+ numUpdatedColumns.setRef(numTargetedColumns);
}
/**
@@ -983,33 +1027,27 @@ public class CatalogOpExecutor {
synchronized(table) {
long newCatalogVersion = catalog_.incrementAndGetCatalogVersion();
catalog_.getLock().writeLock().unlock();
- if (params.getPartition_spec() == null) {
+ if (params.getPartition_set() == null) {
// TODO: Report the number of updated partitions/columns to the user?
+ // TODO: bulk alter the partitions.
dropColumnStats(table);
dropTableStats(table);
} else {
- HdfsPartition partition =
- ((HdfsTable)table).getPartitionFromThriftPartitionSpec(
- params.getPartition_spec());
- if (partition == null) {
- List<String> partitionDescription = Lists.newArrayList();
- for (TPartitionKeyValue v: params.getPartition_spec()) {
- partitionDescription.add(v.name + " = " + v.value);
- }
- throw new ImpalaRuntimeException("Could not find partition: " +
- Joiner.on("/").join(partitionDescription));
- }
+ List<HdfsPartition> partitions =
+ ((HdfsTable)table).getPartitionsFromPartitionSet(params.getPartition_set());
+ if (partitions.isEmpty()) return;
- if (partition.getPartitionStats() != null) {
- PartitionStatsUtil.deletePartStats(partition);
- try {
- applyAlterPartition(table, partition);
- } finally {
- partition.markDirty();
+ for(HdfsPartition partition : partitions) {
+ if (partition.getPartitionStats() != null) {
+ PartitionStatsUtil.deletePartStats(partition);
+ try {
+ applyAlterPartition(table, partition);
+ } finally {
+ partition.markDirty();
+ }
}
}
}
-
loadTableMetadata(table, newCatalogVersion, false, true, null);
addTableToCatalogUpdate(table, resp.result);
} // end of synchronization
@@ -1825,59 +1863,73 @@ public class CatalogOpExecutor {
}
/**
- * Drops an existing partition from the given table in Hive. If the partition is cached,
+ * Drops existing partitions from the given table in Hive. If a partition is cached,
* the associated cache directive will also be removed.
- * Also drops the partition from its Hdfs table.
- * Returns the table object with an updated catalog version. If the partition does not
- * exist and "IfExists" is true, null is returned. If purge is true, partition data is
- * permanently deleted.
+ * Also drops the corresponding partitions from its Hdfs table.
+ * Returns the table object with an updated catalog version. If none of the partitions
+ * exists and "IfExists" is true, null is returned. If purge is true, partition data is
+ * permanently deleted. numUpdatedPartitions is used to inform the client how many
+ * partitions being dropped in this operation.
*/
private Table alterTableDropPartition(Table tbl,
- List<TPartitionKeyValue> partitionSpec, boolean ifExists, boolean purge)
+ List<List<TPartitionKeyValue>> partitionSet,
+ boolean ifExists, boolean purge, Reference<Long> numUpdatedPartitions)
throws ImpalaException {
Preconditions.checkState(Thread.holdsLock(tbl));
+ Preconditions.checkNotNull(partitionSet);
+
TableName tableName = tbl.getTableName();
- if (ifExists && !catalog_.containsHdfsPartition(tableName.getDb(), tableName.getTbl(),
- partitionSpec)) {
- LOG.debug(String.format("Skipping partition drop because (%s) does not exist " +
- "and ifExists is true.", Joiner.on(", ").join(partitionSpec)));
- return null;
+ if (!ifExists) {
+ Preconditions.checkState(!partitionSet.isEmpty());
+ } else {
+ if (partitionSet.isEmpty()) {
+ LOG.debug(String.format("Ignoring empty partition list when dropping " +
+ "partitions from %s because ifExists is true.", tableName));
+ return tbl;
+ }
+ }
+
+ Preconditions.checkArgument(tbl instanceof HdfsTable);
+ List<HdfsPartition> parts =
+ ((HdfsTable) tbl).getPartitionsFromPartitionSet(partitionSet);
+
+ if (!ifExists && parts.isEmpty()) {
+ throw new PartitionNotFoundException(
+ "The partitions being dropped don't exist any more");
}
- HdfsPartition part = catalog_.getHdfsPartition(tableName.getDb(),
- tableName.getTbl(), partitionSpec);
org.apache.hadoop.hive.metastore.api.Table msTbl =
tbl.getMetaStoreTable().deepCopy();
- List<String> values = Lists.newArrayList();
- // Need to add in the values in the same order they are defined in the table.
- for (FieldSchema fs: msTbl.getPartitionKeys()) {
- for (TPartitionKeyValue kv: partitionSpec) {
- if (fs.getName().toLowerCase().equals(kv.getName().toLowerCase())) {
- values.add(kv.getValue());
- }
- }
- }
+
PartitionDropOptions dropOptions = PartitionDropOptions.instance();
dropOptions.purgeData(purge);
+ long numTargetedPartitions = 0L;
try (MetaStoreClient msClient = catalog_.getMetaStoreClient()) {
- msClient.getHiveClient().dropPartition(tableName.getDb(),
- tableName.getTbl(), values, dropOptions);
- updateLastDdlTime(msTbl, msClient);
- if (part.isMarkedCached()) {
- HdfsCachingUtil.uncachePartition(part);
- }
- } catch (NoSuchObjectException e) {
- if (!ifExists) {
- throw new ImpalaRuntimeException(
- String.format(HMS_RPC_ERROR_FORMAT_STR, "dropPartition"), e);
+ for (HdfsPartition part : parts) {
+ try {
+ msClient.getHiveClient().dropPartition(tableName.getDb(), tableName.getTbl(),
+ part.getPartitionValuesAsStrings(true), dropOptions);
+ ++numTargetedPartitions;
+ if (part.isMarkedCached()) {
+ HdfsCachingUtil.uncachePartition(part);
+ }
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new ImpalaRuntimeException(
+ String.format(HMS_RPC_ERROR_FORMAT_STR, "dropPartition"), e);
+ }
+ LOG.debug(
+ String.format("Ignoring '%s' when dropping partitions from %s because" +
+ " ifExists is true.", e, tableName));
+ }
}
- LOG.debug(String.format("Ignoring '%s' when dropping partition from %s because" +
- " ifExists is true.", e, tableName));
+ updateLastDdlTime(msTbl, msClient);
} catch (TException e) {
throw new ImpalaRuntimeException(
String.format(HMS_RPC_ERROR_FORMAT_STR, "dropPartition"), e);
}
- return catalog_.dropPartition(tbl, partitionSpec);
+ numUpdatedPartitions.setRef(numTargetedPartitions);
+ return catalog_.dropPartitions(tbl, partitionSet);
}
/**
@@ -1970,18 +2022,19 @@ public class CatalogOpExecutor {
}
/**
- * Changes the file format for the given table or partition. This is a metadata only
+ * Changes the file format for the given table or partitions. This is a metadata only
* operation, existing table data will not be converted to the new format. After
* changing the file format the table metadata is marked as invalid and will be
* reloaded on the next access.
*/
private boolean alterTableSetFileFormat(Table tbl,
- List<TPartitionKeyValue> partitionSpec, THdfsFileFormat fileFormat)
+ List<List<TPartitionKeyValue>> partitionSet, THdfsFileFormat fileFormat,
+ Reference<Long> numUpdatedPartitions)
throws ImpalaException {
Preconditions.checkState(Thread.holdsLock(tbl));
- Preconditions.checkState(partitionSpec == null || !partitionSpec.isEmpty());
+ Preconditions.checkState(partitionSet == null || !partitionSet.isEmpty());
boolean reloadFileMetadata = false;
- if (partitionSpec == null) {
+ if (partitionSet == null) {
org.apache.hadoop.hive.metastore.api.Table msTbl =
tbl.getMetaStoreTable().deepCopy();
setStorageDescriptorFileFormat(msTbl.getSd(), fileFormat);
@@ -1991,16 +2044,17 @@ public class CatalogOpExecutor {
applyAlterTable(msTbl);
reloadFileMetadata = true;
} else {
- TableName tableName = tbl.getTableName();
- HdfsPartition partition = catalog_.getHdfsPartition(
- tableName.getDb(), tableName.getTbl(), partitionSpec);
- Preconditions.checkNotNull(partition);
- partition.setFileFormat(HdfsFileFormat.fromThrift(fileFormat));
- try {
- applyAlterPartition(tbl, partition);
- } finally {
- partition.markDirty();
+ Preconditions.checkArgument(tbl instanceof HdfsTable);
+ List<HdfsPartition> partitions =
+ ((HdfsTable) tbl).getPartitionsFromPartitionSet(partitionSet);
+ List<HdfsPartition> modifiedParts = Lists.newArrayList();
+ for(HdfsPartition partition: partitions) {
+ partition.setFileFormat(HdfsFileFormat.fromThrift(fileFormat));
+ modifiedParts.add(partition);
}
+ TableName tableName = tbl.getTableName();
+ bulkAlterPartitions(tableName.getDb(), tableName.getTbl(), modifiedParts);
+ numUpdatedPartitions.setRef((long) modifiedParts.size());
}
return reloadFileMetadata;
}
@@ -2024,7 +2078,6 @@ public class CatalogOpExecutor {
private boolean alterTableSetLocation(Table tbl,
List<TPartitionKeyValue> partitionSpec, String location) throws ImpalaException {
Preconditions.checkState(Thread.holdsLock(tbl));
- Preconditions.checkState(partitionSpec == null || !partitionSpec.isEmpty());
boolean reloadFileMetadata = false;
if (partitionSpec == null) {
org.apache.hadoop.hive.metastore.api.Table msTbl =
@@ -2047,35 +2100,44 @@ public class CatalogOpExecutor {
}
/**
- * Appends to the table or partition property metadata for the given table, replacing
+ * Appends to the table or partitions property metadata for the given table, replacing
* the values of any keys that already exist.
*/
private void alterTableSetTblProperties(Table tbl,
- TAlterTableSetTblPropertiesParams params) throws ImpalaException {
+ TAlterTableSetTblPropertiesParams params, Reference<Long> numUpdatedPartitions)
+ throws ImpalaException {
Preconditions.checkState(Thread.holdsLock(tbl));
Map<String, String> properties = params.getProperties();
Preconditions.checkNotNull(properties);
- if (params.isSetPartition_spec()) {
- TableName tableName = tbl.getTableName();
- // Alter partition params.
- HdfsPartition partition = catalog_.getHdfsPartition(
- tableName.getDb(), tableName.getTbl(), params.getPartition_spec());
- switch (params.getTarget()) {
- case TBL_PROPERTY:
- partition.getParameters().putAll(properties);
- break;
- case SERDE_PROPERTY:
- partition.getSerdeInfo().getParameters().putAll(properties);
- break;
- default:
- throw new UnsupportedOperationException(
- "Unknown target TTablePropertyType: " + params.getTarget());
+ if (params.isSetPartition_set()) {
+ Preconditions.checkArgument(tbl instanceof HdfsTable);
+ List<HdfsPartition> partitions =
+ ((HdfsTable) tbl).getPartitionsFromPartitionSet(params.getPartition_set());
+
+ List<HdfsPartition> modifiedParts = Lists.newArrayList();
+ for(HdfsPartition partition: partitions) {
+ switch (params.getTarget()) {
+ case TBL_PROPERTY:
+ partition.getParameters().putAll(properties);
+ break;
+ case SERDE_PROPERTY:
+ partition.getSerdeInfo().getParameters().putAll(properties);
+ break;
+ default:
+ throw new UnsupportedOperationException(
+ "Unknown target TTablePropertyType: " + params.getTarget());
+ }
+ modifiedParts.add(partition);
}
+ TableName tableName = tbl.getTableName();
try {
- applyAlterPartition(tbl, partition);
+ bulkAlterPartitions(tableName.getDb(), tableName.getTbl(), modifiedParts);
} finally {
- partition.markDirty();
+ for (HdfsPartition modifiedPart : modifiedParts) {
+ modifiedPart.markDirty();
+ }
}
+ numUpdatedPartitions.setRef((long) modifiedParts.size());
} else {
// Alter table params.
org.apache.hadoop.hive.metastore.api.Table msTbl =
@@ -2226,56 +2288,68 @@ public class CatalogOpExecutor {
}
/**
- * Caches or uncaches the HDFS location of the target partition and updates the
- * partition's metadata in Hive Metastore Store. If a partition is being cached, the
+ * Caches or uncaches the HDFS location of the target partitions and updates the
+ * partitions' metadata in Hive Metastore Store. If a partition is being cached, the
* partition properties will have the ID of the cache directive added. If the partition
* is being uncached, any outstanding cache directive will be dropped and the cache
* directive ID property key will be cleared.
*/
- private void alterPartitionSetCached(Table tbl, TAlterTableSetCachedParams params)
+ private void alterPartitionSetCached(Table tbl,
+ TAlterTableSetCachedParams params, Reference<Long> numUpdatedPartitions)
throws ImpalaException {
Preconditions.checkState(Thread.holdsLock(tbl));
THdfsCachingOp cacheOp = params.getCache_op();
Preconditions.checkNotNull(cacheOp);
- Preconditions.checkNotNull(params.getPartition_spec());
- // Alter partition params.
+ Preconditions.checkNotNull(params.getPartition_set());
TableName tableName = tbl.getTableName();
- HdfsPartition partition = catalog_.getHdfsPartition(
- tableName.getDb(), tableName.getTbl(), params.getPartition_spec());
+ Preconditions.checkArgument(tbl instanceof HdfsTable);
+ List<HdfsPartition> partitions =
+ ((HdfsTable) tbl).getPartitionsFromPartitionSet(params.getPartition_set());
+ List<HdfsPartition> modifiedParts = Lists.newArrayList();
if (cacheOp.isSet_cached()) {
- // The directive is null if the partition is not cached
- Long directiveId = HdfsCachingUtil.getCacheDirectiveId(
- partition.getParameters());
- short replication = HdfsCachingUtil.getReplicationOrDefault(cacheOp);
- List<Long> cacheDirs = Lists.newArrayList();
-
- if (directiveId == null) {
- cacheDirs.add(HdfsCachingUtil.submitCachePartitionDirective(partition,
- cacheOp.getCache_pool_name(), replication));
- } else {
- if (HdfsCachingUtil.isUpdateOp(cacheOp, partition.getParameters())) {
- HdfsCachingUtil.validateCachePool(cacheOp, directiveId, tableName, partition);
- cacheDirs.add(HdfsCachingUtil.modifyCacheDirective(directiveId, partition,
- cacheOp.getCache_pool_name(), replication));
+ for (HdfsPartition partition : partitions) {
+ // The directive is null if the partition is not cached
+ Long directiveId =
+ HdfsCachingUtil.getCacheDirectiveId(partition.getParameters());
+ short replication = HdfsCachingUtil.getReplicationOrDefault(cacheOp);
+ List<Long> cacheDirs = Lists.newArrayList();
+ if (directiveId == null) {
+ cacheDirs.add(HdfsCachingUtil.submitCachePartitionDirective(
+ partition, cacheOp.getCache_pool_name(), replication));
+ } else {
+ if (HdfsCachingUtil.isUpdateOp(cacheOp, partition.getParameters())) {
+ HdfsCachingUtil.validateCachePool(cacheOp, directiveId, tableName, partition);
+ cacheDirs.add(HdfsCachingUtil.modifyCacheDirective(
+ directiveId, partition, cacheOp.getCache_pool_name(),
+ replication));
+ }
}
- }
- // Once the cache directives are submitted, observe the status of the caching
- // until no more progress is made -- either fully cached or out of cache memory
- if (!cacheDirs.isEmpty()) {
- catalog_.watchCacheDirs(cacheDirs, tableName.toThrift());
+ // Once the cache directives are submitted, observe the status of the caching
+ // until no more progress is made -- either fully cached or out of cache memory
+ if (!cacheDirs.isEmpty()) {
+ catalog_.watchCacheDirs(cacheDirs, tableName.toThrift());
+ }
+ if (!partition.isMarkedCached()) {
+ modifiedParts.add(partition);
+ }
}
-
} else {
- // Partition is not cached, just return.
- if (!partition.isMarkedCached()) return;
- HdfsCachingUtil.uncachePartition(partition);
+ for (HdfsPartition partition : partitions) {
+ if (partition.isMarkedCached()) {
+ HdfsCachingUtil.uncachePartition(partition);
+ modifiedParts.add(partition);
+ }
+ }
}
try {
- applyAlterPartition(tbl, partition);
+ bulkAlterPartitions(tableName.getDb(), tableName.getTbl(), modifiedParts);
} finally {
- partition.markDirty();
+ for (HdfsPartition modifiedPart : modifiedParts) {
+ modifiedPart.markDirty();
+ }
}
+ numUpdatedPartitions.setRef((long) modifiedParts.size());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/service/Frontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index a974278..8657182 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -1203,7 +1203,7 @@ public class Frontend {
Table table = impaladCatalog_.getTable(request.getTable_name().getDb_name(),
request.getTable_name().getTable_name());
if (table instanceof HdfsTable) {
- return ((HdfsTable) table).getFiles(request.getPartition_spec());
+ return ((HdfsTable) table).getFiles(request.getPartition_set());
} else {
throw new InternalException("SHOW FILES only supports Hdfs table. " +
"Unsupported table class: " + table.getClass());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
index b005924..0a78dd5 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeDDLTest.java
@@ -77,21 +77,6 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalysisError("alter table functional_hbase.alltypesagg " + kw +
" partition (i=1)", "Table is not partitioned: functional_hbase.alltypesagg");
- // Duplicate partition key name
- AnalysisError("alter table functional.alltypes " + kw +
- " partition(year=2050, year=2051)", "Duplicate partition key name: year");
- // Not a partition column
- AnalysisError("alter table functional.alltypes " + kw +
- " partition(year=2050, int_col=1)",
- "Column 'int_col' is not a partition column in table: functional.alltypes");
-
- // NULL partition keys
- AnalyzesOk("alter table functional.alltypes " + kw +
- " partition(year=NULL, month=1)");
- AnalyzesOk("alter table functional.alltypes " + kw +
- " partition(year=NULL, month=NULL)");
- AnalyzesOk("alter table functional.alltypes " + kw +
- " partition(year=ascii(null), month=ascii(NULL))");
// Empty string partition keys
AnalyzesOk("alter table functional.insert_string_partitioned " + kw +
" partition(s2='')");
@@ -99,41 +84,13 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalyzesOk("alter table functional.alltypes " + kw +
" partition(year=-1, month=cast((10+5*4) as INT))");
- // Arbitrary exprs as partition key values. Non-constant exprs should fail.
- AnalysisError("alter table functional.alltypes " + kw +
- " partition(year=2050, month=int_col)",
- "Non-constant expressions are not supported as static partition-key values " +
- "in 'month=int_col'.");
- AnalysisError("alter table functional.alltypes " + kw +
- " partition(year=cast(int_col as int), month=12)",
- "Non-constant expressions are not supported as static partition-key values " +
- "in 'year=CAST(int_col AS INT)'.");
-
- // Not a valid column
- AnalysisError("alter table functional.alltypes " + kw +
- " partition(year=2050, blah=1)",
- "Partition column 'blah' not found in table: functional.alltypes");
-
- // Data types don't match
- AnalysisError(
- "alter table functional.insert_string_partitioned " + kw +
- " partition(s2=1234)",
- "Value of partition spec (column=s2) has incompatible type: 'SMALLINT'. " +
- "Expected type: 'STRING'.");
-
- // Loss of precision
- AnalysisError(
- "alter table functional.alltypes " + kw +
- " partition(year=100000000000, month=10)",
- "Partition key value may result in loss of precision.\nWould need to cast" +
- " '100000000000' to 'INT' for partition column: year");
-
-
// Table/Db does not exist
AnalysisError("alter table db_does_not_exist.alltypes " + kw +
- " partition (i=1)", "Database does not exist: db_does_not_exist");
+ " partition (i=1)", "Could not resolve table reference: " +
+ "'db_does_not_exist.alltypes'");
AnalysisError("alter table functional.table_does_not_exist " + kw +
- " partition (i=1)", "Table does not exist: functional.table_does_not_exist");
+ " partition (i=1)", "Could not resolve table reference: " +
+ "'functional.table_does_not_exist'");
// Cannot ALTER TABLE a view.
AnalysisError("alter table functional.alltypes_view " + kw +
@@ -143,8 +100,60 @@ public class AnalyzeDDLTest extends FrontendTestBase {
" partition(year=2050, month=10)",
"ALTER TABLE not allowed on a table produced by a data source: " +
"functional.alltypes_datasource");
+
+ // NULL partition keys
+ AnalyzesOk("alter table functional.alltypes " + kw +
+ " partition(year=NULL, month=1)");
+ AnalyzesOk("alter table functional.alltypes " + kw +
+ " partition(year=NULL, month=NULL)");
+ AnalyzesOk("alter table functional.alltypes " + kw +
+ " partition(year=ascii(null), month=ascii(NULL))");
}
+ // Data types don't match
+ AnalysisError("alter table functional.insert_string_partitioned add" +
+ " partition(s2=1234)",
+ "Value of partition spec (column=s2) has incompatible type: " +
+ "'SMALLINT'. Expected type: 'STRING'.");
+ AnalysisError("alter table functional.insert_string_partitioned drop" +
+ " partition(s2=1234)",
+ "operands of type STRING and SMALLINT are not comparable: s2 = 1234");
+
+ // Loss of precision
+ AnalysisError(
+ "alter table functional.alltypes add " +
+ "partition(year=100000000000, month=10) ",
+ "Partition key value may result in loss of precision.\nWould need to cast " +
+ "'100000000000' to 'INT' for partition column: year");
+
+ // Duplicate partition key name
+ AnalysisError("alter table functional.alltypes add " +
+ "partition(year=2050, year=2051)", "Duplicate partition key name: year");
+
+ // Arbitrary exprs as partition key values. Non-constant exprs should fail.
+ AnalysisError("alter table functional.alltypes add " +
+ "partition(year=2050, month=int_col) ",
+ "Non-constant expressions are not supported as static partition-key " +
+ "values in 'month=int_col'.");
+ AnalysisError("alter table functional.alltypes add " +
+ "partition(year=cast(int_col as int), month=12) ",
+ "Non-constant expressions are not supported as static partition-key " +
+ "values in 'year=CAST(int_col AS INT)'.");
+
+ // Not a partition column
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(year=2050, int_col=1)",
+ "Partition exprs cannot contain non-partition column(s): int_col = 1.");
+
+ // Arbitrary exprs as partition key values. Non-partition columns should fail.
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(year=2050, month=int_col) ",
+ "Partition exprs cannot contain non-partition column(s): month = int_col.");
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(year=cast(int_col as int), month=12) ",
+ "Partition exprs cannot contain non-partition column(s): " +
+ "year = CAST(int_col AS INT).");
+
// IF NOT EXISTS properly checks for partition existence
AnalyzesOk("alter table functional.alltypes add " +
"partition(year=2050, month=10)");
@@ -152,20 +161,63 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"partition(year=2010, month=10)",
"Partition spec already exists: (year=2010, month=10).");
AnalyzesOk("alter table functional.alltypes add if not exists " +
- " partition(year=2010, month=10)");
+ "partition(year=2010, month=10)");
AnalyzesOk("alter table functional.alltypes add if not exists " +
- " partition(year=2010, month=10) location " +
+ "partition(year=2010, month=10) location " +
"'/test-warehouse/alltypes/year=2010/month=10'");
// IF EXISTS properly checks for partition existence
+ // with a fully specified partition.
AnalyzesOk("alter table functional.alltypes drop " +
"partition(year=2010, month=10)");
AnalysisError("alter table functional.alltypes drop " +
"partition(year=2050, month=10)",
- "Partition spec does not exist: (year=2050, month=10).");
+ "No matching partition(s) found.");
AnalyzesOk("alter table functional.alltypes drop if exists " +
"partition(year=2050, month=10)");
+ // NULL partition keys
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(year=NULL, month=1)",
+ "No matching partition(s) found.");
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(year=NULL, month is NULL)",
+ "No matching partition(s) found.");
+
+ // Drop partition using predicates
+ // IF EXISTS is added here
+ AnalyzesOk("alter table functional.alltypes drop " +
+ "partition(year<2011, month!=10)");
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(1=1, month=10)",
+ "Invalid partition expr 1 = 1. " +
+ "A partition spec may not contain constant predicates.");
+ AnalyzesOk("alter table functional.alltypes drop " +
+ "partition(year>1050, month=10)");
+ AnalyzesOk("alter table functional.alltypes drop " +
+ "partition(year>1050 and month=10)");
+ AnalyzesOk("alter table functional.alltypes drop " +
+ "partition(month=10)");
+ AnalyzesOk("alter table functional.alltypes drop " +
+ "partition(month+2000=year)");
+ AnalyzesOk("alter table functional.alltypes drop " +
+ "partition(year>9050, month=10)");
+ AnalyzesOk("alter table functional.alltypes drop if exists " +
+ "partition(year>9050, month=10)");
+
+ // Not a valid column
+ AnalysisError("alter table functional.alltypes add " +
+ "partition(year=2050, blah=1)",
+ "Partition column 'blah' not found in table: functional.alltypes");
+ AnalysisError("alter table functional.alltypes drop " +
+ "partition(year=2050, blah=1)",
+ "Could not resolve column/field reference: 'blah'");
+
+ // Not a partition column
+ AnalysisError("alter table functional.alltypes add " +
+ "partition(year=2050, int_col=1) ",
+ "Column 'int_col' is not a partition column in table: functional.alltypes");
+
// Caching ops
AnalyzesOk("alter table functional.alltypes add " +
"partition(year=2050, month=10) cached in 'testPool'");
@@ -185,25 +237,25 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Valid URIs.
AnalyzesOk("alter table functional.alltypes add " +
- " partition(year=2050, month=10) location " +
+ "partition(year=2050, month=10) location " +
"'/test-warehouse/alltypes/year=2010/month=10'");
AnalyzesOk("alter table functional.alltypes add " +
- " partition(year=2050, month=10) location " +
+ "partition(year=2050, month=10) location " +
"'hdfs://localhost:20500/test-warehouse/alltypes/year=2010/month=10'");
AnalyzesOk("alter table functional.alltypes add " +
- " partition(year=2050, month=10) location " +
+ "partition(year=2050, month=10) location " +
"'s3n://bucket/test-warehouse/alltypes/year=2010/month=10'");
AnalyzesOk("alter table functional.alltypes add " +
- " partition(year=2050, month=10) location " +
+ "partition(year=2050, month=10) location " +
"'file:///test-warehouse/alltypes/year=2010/month=10'");
// Invalid URIs.
AnalysisError("alter table functional.alltypes add " +
- " partition(year=2050, month=10) location " +
+ "partition(year=2050, month=10) location " +
"'foofs://bar/test-warehouse/alltypes/year=2010/month=10'",
"No FileSystem for scheme: foofs");
AnalysisError("alter table functional.alltypes add " +
- " partition(year=2050, month=10) location ' '",
+ "partition(year=2050, month=10) location ' '",
"URI path cannot be empty.");
}
@@ -242,14 +294,19 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Table/Db does not exist
AnalysisError("alter table db_does_not_exist.alltypes add columns (i int)",
- "Database does not exist: db_does_not_exist");
+ "Could not resolve table reference: 'db_does_not_exist.alltypes'");
AnalysisError("alter table functional.table_does_not_exist add columns (i int)",
- "Table does not exist: functional.table_does_not_exist");
+ "Could not resolve table reference: 'functional.table_does_not_exist'");
// Cannot ALTER TABLE a view.
AnalysisError("alter table functional.alltypes_view " +
"add columns (c1 string comment 'hi')",
"ALTER TABLE not allowed on a view: functional.alltypes_view");
+ // Cannot ALTER TABLE a nested collection.
+ AnalysisError("alter table allcomplextypes.int_array_col " +
+ "add columns (c1 string comment 'hi')",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
// Cannot ALTER TABLE produced by a data source.
AnalysisError("alter table functional.alltypes_datasource " +
"add columns (c1 string comment 'hi')",
@@ -278,13 +335,17 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Table/Db does not exist
AnalysisError("alter table db_does_not_exist.alltypes drop column col1",
- "Database does not exist: db_does_not_exist");
+ "Could not resolve table reference: 'db_does_not_exist.alltypes'");
AnalysisError("alter table functional.table_does_not_exist drop column col1",
- "Table does not exist: functional.table_does_not_exist");
+ "Could not resolve table reference: 'functional.table_does_not_exist'");
// Cannot ALTER TABLE a view.
AnalysisError("alter table functional.alltypes_view drop column int_col",
"ALTER TABLE not allowed on a view: functional.alltypes_view");
+ // Cannot ALTER TABLE a nested collection.
+ AnalysisError("alter table allcomplextypes.int_array_col drop column int_col",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
// Cannot ALTER TABLE produced by a data source.
AnalysisError("alter table functional.alltypes_datasource drop column int_col",
"ALTER TABLE not allowed on a table produced by a data source: " +
@@ -324,14 +385,19 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Table/Db does not exist
AnalysisError("alter table db_does_not_exist.alltypes change c1 c2 int",
- "Database does not exist: db_does_not_exist");
+ "Could not resolve table reference: 'db_does_not_exist.alltypes'");
AnalysisError("alter table functional.table_does_not_exist change c1 c2 double",
- "Table does not exist: functional.table_does_not_exist");
+ "Could not resolve table reference: 'functional.table_does_not_exist'");
// Cannot ALTER TABLE a view.
AnalysisError("alter table functional.alltypes_view " +
"change column int_col int_col2 int",
"ALTER TABLE not allowed on a view: functional.alltypes_view");
+ // Cannot ALTER TABLE a nested collection.
+ AnalysisError("alter table allcomplextypes.int_array_col " +
+ "change column int_col int_col2 int",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
// Cannot ALTER TABLE produced by a data source.
AnalysisError("alter table functional.alltypes_datasource " +
"change column int_col int_col2 int",
@@ -353,14 +419,20 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"set location '/a/b'");
AnalyzesOk("alter table functional.alltypes PARTITION (month=11, year=2010) " +
"set fileformat parquetfile");
+ AnalyzesOk("alter table functional.alltypes PARTITION (month<=11, year=2010) " +
+ "set fileformat parquetfile");
AnalyzesOk("alter table functional.stringpartitionkey PARTITION " +
"(string_col='partition1') set fileformat parquet");
AnalyzesOk("alter table functional.stringpartitionkey PARTITION " +
"(string_col='PaRtiTion1') set location '/a/b/c'");
AnalyzesOk("alter table functional.alltypes PARTITION (year=2010, month=11) " +
"set tblproperties('a'='1')");
+ AnalyzesOk("alter table functional.alltypes PARTITION (year<=2010, month=11) " +
+ "set tblproperties('a'='1')");
AnalyzesOk("alter table functional.alltypes PARTITION (year=2010, month=11) " +
"set serdeproperties ('a'='2')");
+ AnalyzesOk("alter table functional.alltypes PARTITION (year<=2010, month=11) " +
+ "set serdeproperties ('a'='2')");
{
// Check that long_properties fail at the analysis layer
@@ -422,33 +494,33 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalyzesOk("alter table functional.alltypes PARTITION " +
"(year=cast(100*20+10 as INT), month=cast(2+9 as INT)) " +
"set location '/a/b'");
- // Arbitrary exprs as partition key values. Non-constant exprs should fail.
- AnalysisError("alter table functional.alltypes PARTITION " +
- "(Year=2050, month=int_col) set fileformat sequencefile",
- "Non-constant expressions are not supported as static partition-key " +
- "values in 'month=int_col'.");
+
+ // Arbitrary exprs as partition key values. One-partition-column-bound exprs are ok.
+ AnalyzesOk("alter table functional.alltypes PARTITION " +
+ "(Year*2=Year+2010, month=11) set fileformat sequencefile");
+
+ // Arbitrary exprs as partition key values. Non-partition-column exprs.
AnalysisError("alter table functional.alltypes PARTITION " +
- "(Year=2050, month=int_col) set location '/a/b'",
- "Non-constant expressions are not supported as static partition-key " +
- "values in 'month=int_col'.");
+ "(int_col=3) set fileformat sequencefile",
+ "Partition exprs cannot contain non-partition column(s): int_col = 3.");
+
+ // Partition expr matches more than one partition in set location statement.
+ AnalysisError("alter table functional.alltypes PARTITION (year!=20) " +
+ "set location '/a/b'",
+ "Partition expr in set location statements can only match " +
+ "one partition. Too many matched partitions year=2009/month=1," +
+ "year=2009/month=2,year=2009/month=3");
// Partition spec does not exist
AnalysisError("alter table functional.alltypes PARTITION (year=2014, month=11) " +
"set location '/a/b'",
- "Partition spec does not exist: (year=2014, month=11)");
+ "No matching partition(s) found.");
AnalysisError("alter table functional.alltypes PARTITION (year=2014, month=11) " +
"set tblproperties('a'='1')",
- "Partition spec does not exist: (year=2014, month=11)");
- AnalysisError("alter table functional.alltypes PARTITION (year=2010) " +
- "set tblproperties('a'='1')",
- "Items in partition spec must exactly match the partition columns " +
- "in the table definition: functional.alltypes (1 vs 2)");
- AnalysisError("alter table functional.alltypes PARTITION (year=2010, year=2010) " +
- "set location '/a/b'",
- "Duplicate partition key name: year");
+ "No matching partition(s) found.");
AnalysisError("alter table functional.alltypes PARTITION (month=11, year=2014) " +
"set fileformat sequencefile",
- "Partition spec does not exist: (month=11, year=2014)");
+ "No matching partition(s) found.");
AnalysisError("alter table functional.alltypesnopart PARTITION (month=1) " +
"set fileformat sequencefile",
"Table is not partitioned: functional.alltypesnopart");
@@ -457,34 +529,32 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"Table is not partitioned: functional.alltypesnopart");
AnalysisError("alter table functional.stringpartitionkey PARTITION " +
"(string_col='partition2') set location '/a/b'",
- "Partition spec does not exist: (string_col='partition2')");
+ "No matching partition(s) found.");
AnalysisError("alter table functional.stringpartitionkey PARTITION " +
"(string_col='partition2') set fileformat sequencefile",
- "Partition spec does not exist: (string_col='partition2')");
+ "No matching partition(s) found.");
AnalysisError("alter table functional.alltypes PARTITION " +
"(year=cast(10*20+10 as INT), month=cast(5*3 as INT)) " +
"set location '/a/b'",
- "Partition spec does not exist: " +
- "(year=CAST(10 * 20 + 10 AS INT), month=CAST(5 * 3 AS INT))");
+ "No matching partition(s) found.");
AnalysisError("alter table functional.alltypes PARTITION " +
"(year=cast(10*20+10 as INT), month=cast(5*3 as INT)) " +
"set fileformat sequencefile",
- "Partition spec does not exist: " +
- "(year=CAST(10 * 20 + 10 AS INT), month=CAST(5 * 3 AS INT))");
+ "No matching partition(s) found.");
// Table/Db does not exist
AnalysisError("alter table db_does_not_exist.alltypes set fileformat sequencefile",
- "Database does not exist: db_does_not_exist");
+ "Could not resolve table reference: 'db_does_not_exist.alltypes'");
AnalysisError("alter table functional.table_does_not_exist set fileformat rcfile",
- "Table does not exist: functional.table_does_not_exist");
+ "Could not resolve table reference: 'functional.table_does_not_exist'");
AnalysisError("alter table db_does_not_exist.alltypes set location '/a/b'",
- "Database does not exist: db_does_not_exist");
+ "Could not resolve table reference: 'db_does_not_exist.alltypes'");
AnalysisError("alter table functional.table_does_not_exist set location '/a/b'",
- "Table does not exist: functional.table_does_not_exist");
+ "Could not resolve table reference: 'functional.table_does_not_exist'");
AnalysisError("alter table functional.no_tbl partition(i=1) set location '/a/b'",
- "Table does not exist: functional.no_tbl");
+ "Could not resolve table reference: 'functional.no_tbl'");
AnalysisError("alter table no_db.alltypes partition(i=1) set fileformat textfile",
- "Database does not exist: no_db");
+ "Could not resolve table reference: 'no_db.alltypes'");
// Valid location
AnalyzesOk("alter table functional.alltypes set location " +
@@ -507,6 +577,10 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Cannot ALTER TABLE a view.
AnalysisError("alter table functional.alltypes_view set fileformat sequencefile",
"ALTER TABLE not allowed on a view: functional.alltypes_view");
+ // Cannot ALTER TABLE a nested collection.
+ AnalysisError("alter table allcomplextypes.int_array_col set fileformat sequencefile",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
// Cannot ALTER TABLE produced by a data source.
AnalysisError("alter table functional.alltypes_datasource set fileformat parquet",
"ALTER TABLE not allowed on a table produced by a data source: " +
@@ -524,6 +598,8 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalyzesOk("alter table functional.alltypes set cached in 'testPool'");
AnalyzesOk("alter table functional.alltypes partition(year=2010, month=12) " +
"set cached in 'testPool'");
+ AnalyzesOk("alter table functional.alltypes partition(year<=2010, month<=12) " +
+ "set cached in 'testPool'");
// Replication factor
AnalyzesOk("alter table functional.alltypes set cached in 'testPool' " +
@@ -542,6 +618,9 @@ public class AnalyzeDDLTest extends FrontendTestBase {
"ALTER TABLE SET not currently supported on HBase tables.");
AnalysisError("alter table functional.view_view set cached in 'testPool'",
"ALTER TABLE not allowed on a view: functional.view_view");
+ AnalysisError("alter table allcomplextypes.int_array_col set cached in 'testPool'",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
AnalysisError("alter table functional.alltypes set cached in 'badPool'",
"The specified cache pool does not exist: badPool");
@@ -565,17 +644,17 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalysisError("alter table functional.alltypestiny partition (year=2009,month=1) " +
"set location '/test-warehouse/new_location'",
"Target partition is cached, please uncache before changing the location " +
- "using: ALTER TABLE functional.alltypestiny PARTITION (year=2009, month=1) " +
+ "using: ALTER TABLE functional.alltypestiny PARTITION (year = 2009, month = 1) " +
"SET UNCACHED");
// Table/db/partition do not exist
AnalysisError("alter table baddb.alltypestiny set cached in 'testPool'",
- "Database does not exist: baddb");
+ "Could not resolve table reference: 'baddb.alltypestiny'");
AnalysisError("alter table functional.badtbl set cached in 'testPool'",
- "Table does not exist: functional.badtbl");
+ "Could not resolve table reference: 'functional.badtbl'");
AnalysisError("alter table functional.alltypestiny partition(year=9999, month=1) " +
"set cached in 'testPool'",
- "Partition spec does not exist: (year=9999, month=1).");
+ "No matching partition(s) found.");
}
@Test
@@ -646,7 +725,7 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Table does not exist.
AnalysisError("alter table bad_tbl set column stats int_col ('numNulls'='2')",
- "Table does not exist: default.bad_tbl");
+ "Could not resolve table reference: 'bad_tbl'");
// Column does not exist.
AnalysisError(
"alter table functional.alltypes set column stats bad_col ('numNulls'='2')",
@@ -656,6 +735,12 @@ public class AnalyzeDDLTest extends FrontendTestBase {
AnalysisError(
"alter table functional.alltypes_view set column stats int_col ('numNulls'='2')",
"ALTER TABLE not allowed on a view: functional.alltypes_view");
+ // Cannot set column stats of a nested collection.
+ AnalysisError(
+ "alter table allcomplextypes.int_array_col " +
+ "set column stats int_col ('numNulls'='2')",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
// Cannot set column stats of partition columns.
AnalysisError(
"alter table functional.alltypes set column stats month ('numDVs'='10')",
@@ -835,6 +920,10 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Cannot ALTER TABLE a view.
AnalysisError("alter table functional.alltypes_view rename to new_alltypes",
"ALTER TABLE not allowed on a view: functional.alltypes_view");
+ // Cannot ALTER TABLE a nested collection.
+ AnalysisError("alter table allcomplextypes.int_array_col rename to new_alltypes",
+ createAnalyzer("functional"),
+ "Database does not exist: allcomplextypes");
// It should be okay to rename an HBase table.
AnalyzesOk("alter table functional_hbase.alltypes rename to new_alltypes");
@@ -847,13 +936,16 @@ public class AnalyzeDDLTest extends FrontendTestBase {
public void TestAlterTableRecoverPartitions() throws CatalogException {
AnalyzesOk("alter table functional.alltypes recover partitions");
AnalysisError("alter table baddb.alltypes recover partitions",
- "Database does not exist: baddb");
+ "Could not resolve table reference: 'baddb.alltypes'");
AnalysisError("alter table functional.badtbl recover partitions",
- "Table does not exist: functional.badtbl");
+ "Could not resolve table reference: 'functional.badtbl'");
AnalysisError("alter table functional.alltypesnopart recover partitions",
"Table is not partitioned: functional.alltypesnopart");
AnalysisError("alter table functional.view_view recover partitions",
"ALTER TABLE not allowed on a view: functional.view_view");
+ AnalysisError("alter table allcomplextypes.int_array_col recover partitions",
+ createAnalyzer("functional"),
+ "ALTER TABLE not allowed on a nested collection: allcomplextypes.int_array_col");
AnalysisError("alter table functional_hbase.alltypes recover partitions",
"ALTER TABLE RECOVER PARTITIONS must target an HDFS table: " +
"functional_hbase.alltypes");
@@ -958,10 +1050,10 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Cannot compute stats on a database.
AnalysisError("compute stats tbl_does_not_exist",
- "Table does not exist: default.tbl_does_not_exist");
+ "Could not resolve table reference: 'tbl_does_not_exist'");
// Cannot compute stats on a view.
AnalysisError("compute stats functional.alltypes_view",
- "COMPUTE STATS not supported for view functional.alltypes_view");
+ "COMPUTE STATS not supported for view: functional.alltypes_view");
AnalyzesOk("compute stats functional_avro_snap.alltypes");
// Test mismatched column definitions and Avro schema (HIVE-6308, IMPALA-867).
@@ -991,28 +1083,26 @@ public class AnalyzeDDLTest extends FrontendTestBase {
checkComputeStatsStmt("compute incremental stats functional.alltypes");
checkComputeStatsStmt(
"compute incremental stats functional.alltypes partition(year=2010, month=10)");
+ checkComputeStatsStmt(
+ "compute incremental stats functional.alltypes partition(year<=2010)");
AnalysisError(
"compute incremental stats functional.alltypes partition(year=9999, month=10)",
- "Partition spec does not exist: (year=9999, month=10)");
- AnalysisError(
- "compute incremental stats functional.alltypes partition(year=2010)",
- "Items in partition spec must exactly match the partition columns in the table " +
- "definition: functional.alltypes (1 vs 2)");
+ "No matching partition(s) found.");
AnalysisError(
"compute incremental stats functional.alltypes partition(year=2010, month)",
- "Syntax error");
+ "Partition expr requires return type 'BOOLEAN'. Actual type is 'INT'.");
// Test that NULL partitions generates a valid query
checkComputeStatsStmt("compute incremental stats functional.alltypesagg " +
- "partition(year=2010, month=1, day=NULL)");
+ "partition(year=2010, month=1, day is NULL)");
AnalysisError("compute incremental stats functional_hbase.alltypes " +
"partition(year=2010, month=1)", "COMPUTE INCREMENTAL ... PARTITION not " +
"supported for non-HDFS table functional_hbase.alltypes");
AnalysisError("compute incremental stats functional.view_view",
- "COMPUTE STATS not supported for view functional.view_view");
+ "COMPUTE STATS not supported for view: functional.view_view");
}
@@ -1020,9 +1110,11 @@ public class AnalyzeDDLTest extends FrontendTestBase {
public void TestDropIncrementalStats() throws AnalysisException {
AnalyzesOk(
"drop incremental stats functional.alltypes partition(year=2010, month=10)");
+ AnalyzesOk(
+ "drop incremental stats functional.alltypes partition(year<=2010, month=10)");
AnalysisError(
"drop incremental stats functional.alltypes partition(year=9999, month=10)",
- "Partition spec does not exist: (year=9999, month=10)");
+ "No matching partition(s) found.");
}
@@ -1032,10 +1124,10 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Table does not exist
AnalysisError("drop stats tbl_does_not_exist",
- "Table does not exist: default.tbl_does_not_exist");
+ "Could not resolve table reference: 'tbl_does_not_exist'");
// Database does not exist
AnalysisError("drop stats no_db.no_tbl",
- "Database does not exist: no_db");
+ "Could not resolve table reference: 'no_db.no_tbl'");
AnalysisError("drop stats functional.alltypes partition(year=2010, month=10)",
"Syntax error");
@@ -2959,32 +3051,38 @@ public class AnalyzeDDLTest extends FrontendTestBase {
// Test empty table
AnalyzesOk(String.format("show files in functional.emptytable"));
- String[] partitions = new String[] { "", "partition(month=10, year=2010)" };
+ String[] partitions = new String[] {
+ "",
+ "partition(month=10, year=2010)",
+ "partition(month>10, year<2011, year>2008)"};
for (String partition: partitions) {
AnalyzesOk(String.format("show files in functional.alltypes %s", partition));
// Database/table doesn't exist.
AnalysisError(String.format("show files in baddb.alltypes %s", partition),
- "Database does not exist: baddb");
+ "Could not resolve table reference: 'baddb.alltypes'");
AnalysisError(String.format("show files in functional.badtbl %s", partition),
- "Table does not exist: functional.badtbl");
+ "Could not resolve table reference: 'functional.badtbl'");
// Cannot show files on a non hdfs table.
AnalysisError(String.format("show files in functional.alltypes_view %s",
partition),
"SHOW FILES not applicable to a non hdfs table: functional.alltypes_view");
+ AnalysisError(String.format("show files in allcomplextypes.int_array_col %s",
+ partition), createAnalyzer("functional"),
+ "SHOW FILES not applicable to a non hdfs table: allcomplextypes.int_array_col");
}
// Not a partition column.
AnalysisError("show files in functional.alltypes partition(year=2010,int_col=1)",
- "Column 'int_col' is not a partition column in table: functional.alltypes");
+ "Partition exprs cannot contain non-partition column(s): int_col = 1.");
// Not a valid column.
AnalysisError("show files in functional.alltypes partition(year=2010,day=1)",
- "Partition column 'day' not found in table: functional.alltypes");
+ "Could not resolve column/field reference: 'day'");
// Table is not partitioned.
AnalysisError("show files in functional.tinyinttable partition(int_col=1)",
"Table is not partitioned: functional.tinyinttable");
// Partition spec does not exist
AnalysisError("show files in functional.alltypes partition(year=2010,month=NULL)",
- "Partition spec does not exist: (year=2010, month=NULL)");
+ "No matching partition(s) found.");
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
index a6760de..cacdd83 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeStmtsTest.java
@@ -3466,7 +3466,7 @@ public class AnalyzeStmtsTest extends AnalyzerTest {
testNumberOfMembers(ValuesStmt.class, 0);
// Also check TableRefs.
- testNumberOfMembers(TableRef.class, 18);
+ testNumberOfMembers(TableRef.class, 19);
testNumberOfMembers(BaseTableRef.class, 0);
testNumberOfMembers(InlineViewRef.class, 8);
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index 0affd4e..69c90da 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -1768,8 +1768,8 @@ public class ParserTest extends FrontendTestBase {
ParserError("SHOW TABLE STATS 'strlit'");
// Missing table.
ParserError("SHOW FILES IN");
- // Invalid partition.
- ParserError("SHOW FILES IN db.tbl PARTITION(p)");
+
+ ParsesOk("SHOW FILES IN db.tbl PARTITION(p)");
}
@Test
@@ -2002,9 +2002,6 @@ public class ParserTest extends FrontendTestBase {
ParsesOk("ALTER TABLE Foo ADD PARTITION (i=NULL, j=2, k=NULL)");
ParsesOk("ALTER TABLE Foo ADD PARTITION (i=abc, j=(5*8+10), k=!true and false)");
- // Cannot use dynamic partition syntax
- ParserError("ALTER TABLE TestDb.Foo ADD PARTITION (partcol)");
- ParserError("ALTER TABLE TestDb.Foo ADD PARTITION (i=1, partcol)");
// Location needs to be a string literal
ParserError("ALTER TABLE TestDb.Foo ADD PARTITION (i=1, s='Hello') LOCATION a/b");
@@ -2070,10 +2067,6 @@ public class ParserTest extends FrontendTestBase {
ParsesOk(String.format("ALTER TABLE Foo DROP PARTITION (i=abc, "
+ "j=(5*8+10), k=!true and false) %s", kw));
- // Cannot use dynamic partition syntax
- ParserError(String.format("ALTER TABLE Foo DROP PARTITION (partcol) %s", kw));
- ParserError(String.format("ALTER TABLE Foo DROP PARTITION (i=1, j) %s", kw));
-
ParserError(String.format("ALTER TABLE Foo DROP IF NOT EXISTS "
+ "PARTITION (i=1, s='Hello') %s", kw));
ParserError(String.format("ALTER TABLE TestDb.Foo DROP (i=1, s='Hello') %s", kw));
@@ -2126,10 +2119,7 @@ public class ParserTest extends FrontendTestBase {
ParsesOk("ALTER TABLE Foo PARTITION (i=1,s='str') SET LOCATION '/a/i=1/s=str'");
ParsesOk("ALTER TABLE Foo PARTITION (s='str') SET LOCATION '/a/i=1/s=str'");
- ParserError("ALTER TABLE Foo PARTITION (s) SET LOCATION '/a'");
ParserError("ALTER TABLE Foo PARTITION () SET LOCATION '/a'");
- ParserError("ALTER TABLE Foo PARTITION ('str') SET FILEFORMAT TEXTFILE");
- ParserError("ALTER TABLE Foo PARTITION (a=1, 5) SET FILEFORMAT TEXTFILE");
ParserError("ALTER TABLE Foo PARTITION () SET FILEFORMAT PARQUETFILE");
ParserError("ALTER TABLE Foo PARTITION (,) SET FILEFORMAT PARQUET");
ParserError("ALTER TABLE Foo PARTITION (a=1) SET FILEFORMAT");
@@ -3331,13 +3321,10 @@ public class ParserTest extends FrontendTestBase {
ParsesOk(
"COMPUTE INCREMENTAL STATS functional.alltypes PARTITION(month=10, year=2010)");
- // No dynamic partition specs
- ParserError("COMPUTE INCREMENTAL STATS functional.alltypes PARTITION(month, year)");
ParserError("COMPUTE INCREMENTAL STATS");
ParsesOk("DROP INCREMENTAL STATS functional.alltypes PARTITION(month=10, year=2010)");
- ParserError("DROP INCREMENTAL STATS functional.alltypes PARTITION(month, year)");
ParserError("DROP INCREMENTAL STATS functional.alltypes");
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/shell/impala_client.py
----------------------------------------------------------------------
diff --git a/shell/impala_client.py b/shell/impala_client.py
index f57a015..f84c14e 100755
--- a/shell/impala_client.py
+++ b/shell/impala_client.py
@@ -450,7 +450,7 @@ class ImpalaClient(object):
def expect_result_metadata(self, query_str):
""" Given a query string, return True if impalad expects result metadata"""
- excluded_query_types = ['use', 'alter', 'drop']
+ excluded_query_types = ['use', 'drop']
if True in set(map(query_str.startswith, excluded_query_types)):
return False
return True
[4/9] incubator-impala git commit: IMPALA-1654: General partition
exprs in DDL operations.
Posted by sa...@apache.org.
IMPALA-1654: General partition exprs in DDL operations.
This commit handles partition related DDL in a more general way. We can
now use compound predicates to specify a list of partitions in
statements like ALTER TABLE DROP PARTITION and COMPUTE INCREMENTAL
STATS, etc. It will also make sure some statements only accept one
partition at a time, such as PARTITION SET LOCATION and LOAD DATA. ALTER
TABLE ADD PARTITION remains using the old PartitionKeyValue's logic.
The changed partition related DDLs are as follows,
Table: p (i int) partitioned by (j int, k string)
Partitions:
+-------+---+-------+--------+------+--------------+-------------------+
| j | k | #Rows | #Files | Size | Bytes Cached | Cache Replication |
+-------+---+-------+--------+------+--------------+-------------------+
| 1 | a | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| 1 | b | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| 1 | c | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| 2 | d | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| 2 | e | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| 2 | f | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| Total | | -1 | 0 | 0B | 0B | |
+-------+---+-------+--------+------+--------------+-------------------+
1. show files in p partition (j<2, k='a');
2. alter table p partition (j<2, k in ("b","c") set cached in 'testPool';
// j can appear more than once,
3.1. alter table p partition (j<2, j>0, k<>"d") set uncached;
// it is the same as
3.2. alter table p partition (j<2 and j>0, not k="e") set uncached;
// we can also do 'or'
3.3. alter table p partition (j<2 or j>0, k like "%") set uncached;
// missing 'k' matches all values of k
4. alter table p partition (j<2) set fileformat textfile;
5. alter table p partition (k rlike ".*") set serdeproperties ("k"="v");
6. alter table p partition (j is not null) set tblproperties ("k"="v");
7. alter table p drop partition (j<2);
8. compute incremental stats p partition(j<2);
The remaining old partition related DDLs are as follows,
1. load data inpath '/path/from' into table p partition (j=2, k="d");
2. alter table p add partition (j=2, k="g");
3. alter table p partition (j=2, k="g") set location '/path/to';
4. insert into p partition (j=2, k="g") values (1), (2), (3);
General partition expressions or partially specified partition specs
allows partition predicates to return empty partition set no matter
'IF EXISTS' is specified.
Examples:
[localhost.localdomain:21000] >
alter table p drop partition (j=2, k="f");
Query: alter table p drop partition (j=2, k="f")
+-------------------------+
| summary |
+-------------------------+
| Dropped 1 partition(s). |
+-------------------------+
Fetched 1 row(s) in 0.78s
[localhost.localdomain:21000] >
alter table p drop partition (j=2, k<"f");
Query: alter table p drop partition (j=2, k<"f")
+-------------------------+
| summary |
+-------------------------+
| Dropped 2 partition(s). |
+-------------------------+
Fetched 1 row(s) in 0.41s
[localhost.localdomain:21000] >
alter table p drop partition (k="a");
Query: alter table p drop partition (k="a")
+-------------------------+
| summary |
+-------------------------+
| Dropped 1 partition(s). |
+-------------------------+
Fetched 1 row(s) in 0.25s
[localhost.localdomain:21000] > show partitions p;
Query: show partitions p
+-------+---+-------+--------+------+--------------+-------------------+
| j | k | #Rows | #Files | Size | Bytes Cached | Cache Replication |
+-------+---+-------+--------+------+--------------+-------------------+
| 1 | b | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| 1 | c | -1 | 0 | 0B | NOT CACHED | NOT CACHED |
| Total | | -1 | 0 | 0B | 0B | |
+-------+---+-------+--------+------+--------------+-------------------+
Fetched 3 row(s) in 0.01s
Change-Id: I2c9162fcf9d227b8daf4c2e761d57bab4e26408f
Reviewed-on: http://gerrit.cloudera.org:8080/3942
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/628685ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/628685ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/628685ae
Branch: refs/heads/master
Commit: 628685ae749daa133c9b7a6184c5523136979974
Parents: 3f2f008
Author: Amos Bird <am...@gmail.com>
Authored: Wed Nov 25 06:20:15 2015 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 03:27:36 2016 +0000
----------------------------------------------------------------------
be/src/service/query-exec-state.cc | 19 +-
be/src/service/query-exec-state.h | 1 +
common/thrift/CatalogService.thrift | 2 +-
common/thrift/Frontend.thrift | 6 +-
common/thrift/JniCatalog.thrift | 24 +-
fe/src/main/cup/sql-parser.cup | 62 ++-
.../analysis/AlterTableDropPartitionStmt.java | 22 +-
.../analysis/AlterTableSetCachedStmt.java | 30 +-
.../analysis/AlterTableSetFileFormatStmt.java | 12 +-
.../analysis/AlterTableSetLocationStmt.java | 50 ++-
.../impala/analysis/AlterTableSetStmt.java | 18 +-
.../analysis/AlterTableSetTblProperties.java | 8 +-
.../apache/impala/analysis/AlterTableStmt.java | 24 +-
.../org/apache/impala/analysis/Analyzer.java | 24 +-
.../apache/impala/analysis/BaseTableRef.java | 2 +-
.../impala/analysis/CollectionTableRef.java | 2 +-
.../impala/analysis/ComputeStatsStmt.java | 105 +++--
.../apache/impala/analysis/DropStatsStmt.java | 55 ++-
.../apache/impala/analysis/InlineViewRef.java | 5 +-
.../apache/impala/analysis/PartitionSet.java | 199 +++++++++
.../apache/impala/analysis/PartitionSpec.java | 62 +--
.../impala/analysis/PartitionSpecBase.java | 94 +++++
.../apache/impala/analysis/ShowFilesStmt.java | 44 +-
.../org/apache/impala/analysis/TableRef.java | 12 +-
.../apache/impala/analysis/TupleDescriptor.java | 14 +
.../impala/catalog/CatalogServiceCatalog.java | 19 +
.../org/apache/impala/catalog/HdfsTable.java | 80 +++-
.../java/org/apache/impala/catalog/Table.java | 4 +
.../impala/planner/HdfsPartitionPruner.java | 19 +-
.../impala/planner/SingleNodePlanner.java | 2 +-
.../impala/service/CatalogOpExecutor.java | 402 +++++++++++--------
.../org/apache/impala/service/Frontend.java | 2 +-
.../apache/impala/analysis/AnalyzeDDLTest.java | 334 +++++++++------
.../impala/analysis/AnalyzeStmtsTest.java | 2 +-
.../org/apache/impala/analysis/ParserTest.java | 17 +-
shell/impala_client.py | 2 +-
.../queries/QueryTest/alter-table.test | 45 +++
.../queries/QueryTest/compute-stats.test | 2 +
.../queries/QueryTest/hdfs-caching.test | 2 +-
.../queries/QueryTest/kudu_alter.test | 6 +
.../QueryTest/partition-ddl-predicates.test | 156 +++++++
tests/metadata/test_ddl.py | 4 +
42 files changed, 1398 insertions(+), 596 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/be/src/service/query-exec-state.cc
----------------------------------------------------------------------
diff --git a/be/src/service/query-exec-state.cc b/be/src/service/query-exec-state.cc
index bfb5a86..6075914 100644
--- a/be/src/service/query-exec-state.cc
+++ b/be/src/service/query-exec-state.cc
@@ -516,6 +516,9 @@ Status ImpalaServer::QueryExecState::ExecDdlRequest() {
DCHECK(exec_request_.__isset.query_exec_request);
RETURN_IF_ERROR(ExecQueryOrDmlRequest(exec_request_.query_exec_request));
}
+
+ // Set the results to be reported to the client.
+ SetResultSet(catalog_op_executor_->ddl_exec_response());
return Status::OK();
}
@@ -919,6 +922,13 @@ Status ImpalaServer::QueryExecState::UpdateCatalog() {
return Status::OK();
}
+void ImpalaServer::QueryExecState::SetResultSet(const TDdlExecResponse* ddl_resp) {
+ if (ddl_resp != NULL && ddl_resp->__isset.result_set) {
+ result_metadata_ = ddl_resp->result_set.schema;
+ request_result_set_.reset(new vector<TResultRow>(ddl_resp->result_set.rows));
+ }
+}
+
void ImpalaServer::QueryExecState::SetResultSet(const vector<string>& results) {
request_result_set_.reset(new vector<TResultRow>);
request_result_set_->resize(results.size());
@@ -1026,14 +1036,7 @@ Status ImpalaServer::QueryExecState::UpdateTableAndColumnStats(
exec_request_.query_options.sync_ddl));
// Set the results to be reported to the client.
- const TDdlExecResponse* ddl_resp = catalog_op_executor_->ddl_exec_response();
- if (ddl_resp != NULL && ddl_resp->__isset.result_set) {
- result_metadata_ = ddl_resp->result_set.schema;
- request_result_set_.reset(new vector<TResultRow>);
- request_result_set_->assign(
- ddl_resp->result_set.rows.begin(), ddl_resp->result_set.rows.end());
- }
-
+ SetResultSet(catalog_op_executor_->ddl_exec_response());
query_events_->MarkEvent("Metastore update finished");
return Status::OK();
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/be/src/service/query-exec-state.h
----------------------------------------------------------------------
diff --git a/be/src/service/query-exec-state.h b/be/src/service/query-exec-state.h
index 55e3ffa..28c8cba 100644
--- a/be/src/service/query-exec-state.h
+++ b/be/src/service/query-exec-state.h
@@ -374,6 +374,7 @@ class ImpalaServer::QueryExecState {
/// Copies results into request_result_set_
/// TODO: Have the FE return list<Data.TResultRow> so that this isn't necessary
+ void SetResultSet(const TDdlExecResponse* ddl_resp);
void SetResultSet(const std::vector<std::string>& results);
void SetResultSet(const std::vector<std::string>& col1,
const std::vector<std::string>& col2);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/common/thrift/CatalogService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift
index ec4bde2..188f71a 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -145,7 +145,7 @@ struct TDdlExecResponse {
2: optional bool new_table_created;
// Result of DDL operation to be returned to the client. Currently only set
- // by COMPUTE STATS.
+ // by COMPUTE STATS and ALTER TABLE.
3: optional Results.TResultSet result_set
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/common/thrift/Frontend.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index 8c279f5..6fdf972 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -209,9 +209,9 @@ struct TShowTablesParams {
struct TShowFilesParams {
1: required CatalogObjects.TTableName table_name
- // An optional partition spec. Set if this operation should apply to a specific
- // partition rather than the base table.
- 2: optional list<CatalogObjects.TPartitionKeyValue> partition_spec
+ // An optional partition set. Set if this operation should apply to a list of
+ // partitions rather than the base table.
+ 2: optional list<list<CatalogObjects.TPartitionKeyValue>> partition_set
}
// Parameters for SHOW [CURRENT] ROLES and SHOW ROLE GRANT GROUP <groupName> commands
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/common/thrift/JniCatalog.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/JniCatalog.thrift b/common/thrift/JniCatalog.thrift
index 8ed9fe3..7fa7f7b 100644
--- a/common/thrift/JniCatalog.thrift
+++ b/common/thrift/JniCatalog.thrift
@@ -111,10 +111,10 @@ struct TDropStatsParams {
// Fully qualified name of the target table
1: required CatalogObjects.TTableName table_name
- // If set, delete the stats only for a particular partition, but do not recompute the
+ // If set, delete the stats only for specified partitions, but do not recompute the
// stats for the whole table. This is set only for
// DROP INCREMENTAL STATS <table> PARTITION(...)
- 2: optional list<CatalogObjects.TPartitionKeyValue> partition_spec
+ 2: optional list<list<CatalogObjects.TPartitionKeyValue>> partition_set
}
// Parameters of CREATE FUNCTION commands
@@ -194,8 +194,8 @@ struct TAlterTableDropColParams {
// Parameters for ALTER TABLE DROP PARTITION commands
struct TAlterTableDropPartitionParams {
- // The partition spec (list of keys and values) to add.
- 1: required list<CatalogObjects.TPartitionKeyValue> partition_spec
+ // The partition set used to drop partitions.
+ 1: required list<list<CatalogObjects.TPartitionKeyValue>> partition_set
// If true, no error is raised if no partition with the specified spec exists.
2: required bool if_exists
@@ -222,18 +222,18 @@ struct TAlterTableSetTblPropertiesParams {
// Map of property names to property values.
2: required map<string, string> properties
- // If set, alters the properties of the given partition, otherwise
+ // If set, alters the properties of the given partitions, otherwise
// those of the table.
- 3: optional list<CatalogObjects.TPartitionKeyValue> partition_spec
+ 3: optional list<list<CatalogObjects.TPartitionKeyValue>> partition_set
}
-// Parameters for ALTER TABLE SET [PARTITION partitionSpec] FILEFORMAT commands.
+// Parameters for ALTER TABLE SET [PARTITION partitionSet] FILEFORMAT commands.
struct TAlterTableSetFileFormatParams {
// New file format.
1: required CatalogObjects.THdfsFileFormat file_format
- // An optional partition spec, set if modifying the fileformat of a partition.
- 2: optional list<CatalogObjects.TPartitionKeyValue> partition_spec
+ // An optional partition set, set if modifying the fileformat of the partitions.
+ 2: optional list<list<CatalogObjects.TPartitionKeyValue>> partition_set
}
// Parameters for ALTER TABLE SET [PARTITION partitionSpec] location commands.
@@ -270,14 +270,14 @@ struct TAlterTableUpdateStatsParams {
6: optional bool is_incremental
}
-// Parameters for ALTER TABLE SET [PARTITION partitionSpec] CACHED|UNCACHED
+// Parameters for ALTER TABLE SET [PARTITION partitionSet] CACHED|UNCACHED
struct TAlterTableSetCachedParams {
// Details on what operation to perform (cache or uncache)
1: required THdfsCachingOp cache_op
- // An optional partition spec, set if marking a partition as cached/uncached
+ // An optional partition set, set if marking the partitions as cached/uncached
// rather than a table.
- 2: optional list<CatalogObjects.TPartitionKeyValue> partition_spec
+ 2: optional list<list<CatalogObjects.TPartitionKeyValue>> partition_set
}
// Parameters for all ALTER TABLE commands.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 57d5a24..1884036 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -370,6 +370,10 @@ nonterminal StatementBase explain_stmt;
nonterminal PartitionSpec opt_partition_spec;
// Required partition spec
nonterminal PartitionSpec partition_spec;
+// Optional partition set
+nonterminal PartitionSet opt_partition_set;
+// Required partition set
+nonterminal PartitionSet partition_set;
nonterminal ArrayList<PartitionKeyValue> partition_clause;
nonterminal ArrayList<PartitionKeyValue> static_partition_key_value_list;
nonterminal ArrayList<PartitionKeyValue> partition_key_value_list;
@@ -904,34 +908,39 @@ alter_tbl_stmt ::=
column_def:col_def
{: RESULT = new AlterTableChangeColStmt(table, col_name, col_def); :}
| KW_ALTER KW_TABLE table_name:table KW_DROP if_exists_val:if_exists
- partition_spec:partition purge_val:purge
- {: RESULT = new AlterTableDropPartitionStmt(table, partition, if_exists, purge); :}
- | KW_ALTER KW_TABLE table_name:table opt_partition_spec:partition KW_SET KW_FILEFORMAT
+ partition_set:partitions purge_val:purge
+ {: RESULT = new AlterTableDropPartitionStmt(table, partitions, if_exists, purge); :}
+ | KW_ALTER KW_TABLE table_name:table opt_partition_set:partitions KW_SET KW_FILEFORMAT
file_format_val:file_format
- {: RESULT = new AlterTableSetFileFormatStmt(table, partition, file_format); :}
- | KW_ALTER KW_TABLE table_name:table opt_partition_spec:partition KW_SET
+ {: RESULT = new AlterTableSetFileFormatStmt(table, partitions, file_format); :}
+ | KW_ALTER KW_TABLE table_name:table opt_partition_set:partitions KW_SET
KW_LOCATION STRING_LITERAL:location
- {: RESULT = new AlterTableSetLocationStmt(table, partition, new HdfsUri(location)); :}
+ {:
+ // Need to check in analysis that the partition set only matches a single partition.
+ // Avoids a reduce/reduce conflict and allows the user to select a partition without
+ // fully specifying all partition-key values.
+ RESULT = new AlterTableSetLocationStmt(table, partitions, new HdfsUri(location));
+ :}
| KW_ALTER KW_TABLE table_name:table KW_RENAME KW_TO table_name:new_table
{: RESULT = new AlterTableOrViewRenameStmt(table, new_table, true); :}
- | KW_ALTER KW_TABLE table_name:table opt_partition_spec:partition KW_SET
+ | KW_ALTER KW_TABLE table_name:table opt_partition_set:partitions KW_SET
table_property_type:target LPAREN properties_map:properties RPAREN
- {: RESULT = new AlterTableSetTblProperties(table, partition, target, properties); :}
- | KW_ALTER KW_TABLE table_name:table opt_partition_spec:partition KW_SET
+ {: RESULT = new AlterTableSetTblProperties(table, partitions, target, properties); :}
+ | KW_ALTER KW_TABLE table_name:table opt_partition_set:partition KW_SET
KW_COLUMN KW_STATS IDENT:col LPAREN properties_map:map RPAREN
{:
- // The opt_partition_spec is used to avoid conflicts even though
+ // The opt_partition_set is used to avoid conflicts even though
// a partition clause does not make sense for this stmt. If a partition
// is given, manually throw a parse error.
if (partition != null) parser.parseError("set", SqlParserSymbols.KW_SET);
RESULT = new AlterTableSetColumnStats(table, col, map);
:}
- | KW_ALTER KW_TABLE table_name:table opt_partition_spec:partition KW_SET
+ | KW_ALTER KW_TABLE table_name:table opt_partition_set:partitions KW_SET
cache_op_val:cache_op
{:
// Ensure a parser error is thrown for ALTER statements if no cache op is specified.
if (cache_op == null) parser.parseError("set", SqlParserSymbols.KW_SET);
- RESULT = new AlterTableSetCachedStmt(table, partition, cache_op);
+ RESULT = new AlterTableSetCachedStmt(table, partitions, cache_op);
:}
| KW_ALTER KW_TABLE table_name:table KW_RECOVER KW_PARTITIONS
{: RESULT = new AlterTableRecoverPartitionsStmt(table); :}
@@ -1550,15 +1559,15 @@ compute_stats_stmt ::=
{: RESULT = new ComputeStatsStmt(table); :}
| KW_COMPUTE KW_INCREMENTAL KW_STATS table_name:table
{: RESULT = new ComputeStatsStmt(table, true, null); :}
- | KW_COMPUTE KW_INCREMENTAL KW_STATS table_name:table partition_spec:spec
- {: RESULT = new ComputeStatsStmt(table, true, spec); :}
+ | KW_COMPUTE KW_INCREMENTAL KW_STATS table_name:table partition_set:partitions
+ {: RESULT = new ComputeStatsStmt(table, true, partitions); :}
;
drop_stats_stmt ::=
KW_DROP KW_STATS table_name:table
{: RESULT = new DropStatsStmt(table); :}
- | KW_DROP KW_INCREMENTAL KW_STATS table_name:table partition_spec:spec
- {: RESULT = new DropStatsStmt(table, spec); :}
+ | KW_DROP KW_INCREMENTAL KW_STATS table_name:table partition_set:partitions
+ {: RESULT = new DropStatsStmt(table, partitions); :}
;
drop_db_stmt ::=
@@ -1626,6 +1635,23 @@ partition_key_value_list ::=
:}
;
+// TODO: reuse this for INSERT statement.
+// A partition set is a set of expressions used to select a list of partitions
+// for certain operation such as DROP PARTITION. This is different than a partition
+// clause in an INSERT statement. Partition clause contains dynamic and static
+// partition key/values.
+partition_set ::=
+ KW_PARTITION LPAREN expr_list:list RPAREN
+ {: RESULT = new PartitionSet(list); :}
+ ;
+
+opt_partition_set ::=
+ partition_set:partition_set
+ {: RESULT = partition_set; :}
+ | /* Empty */
+ {: RESULT = null; :}
+ ;
+
// A partition spec is a set of static partition key/value pairs. This is a bit
// different than a partition clause in an INSERT statement because that allows
// for dynamic and static partition key/values.
@@ -2020,8 +2046,8 @@ show_create_function_stmt ::=
;
show_files_stmt ::=
- KW_SHOW KW_FILES KW_IN table_name:table opt_partition_spec:partition
- {: RESULT = new ShowFilesStmt(table, partition); :}
+ KW_SHOW KW_FILES KW_IN table_name:table opt_partition_set:partitions
+ {: RESULT = new ShowFilesStmt(table, partitions); :}
;
describe_db_stmt ::=
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
index 773fea4..9eb580d 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableDropPartitionStmt.java
@@ -29,18 +29,18 @@ import com.google.common.base.Preconditions;
*/
public class AlterTableDropPartitionStmt extends AlterTableStmt {
private final boolean ifExists_;
- private final PartitionSpec partitionSpec_;
+ private final PartitionSet partitionSet_;
// Setting this value causes dropped partition(s) to be permanently
// deleted. For example, for HDFS tables it skips the trash mechanism
private final boolean purgePartition_;
public AlterTableDropPartitionStmt(TableName tableName,
- PartitionSpec partitionSpec, boolean ifExists, boolean purgePartition) {
+ PartitionSet partitionSet, boolean ifExists, boolean purgePartition) {
super(tableName);
- Preconditions.checkNotNull(partitionSpec);
- partitionSpec_ = partitionSpec;
- partitionSpec_.setTableName(tableName);
+ Preconditions.checkNotNull(partitionSet);
+ partitionSet_ = partitionSet;
+ partitionSet_.setTableName(tableName);
ifExists_ = ifExists;
purgePartition_ = purgePartition;
}
@@ -52,7 +52,7 @@ public class AlterTableDropPartitionStmt extends AlterTableStmt {
StringBuilder sb = new StringBuilder("ALTER TABLE " + getTbl());
sb.append(" DROP ");
if (ifExists_) sb.append("IF EXISTS ");
- sb.append(" DROP " + partitionSpec_.toSql());
+ sb.append(" DROP " + partitionSet_.toSql());
if (purgePartition_) sb.append(" PURGE");
return sb.toString();
}
@@ -62,8 +62,8 @@ public class AlterTableDropPartitionStmt extends AlterTableStmt {
TAlterTableParams params = super.toThrift();
params.setAlter_type(TAlterTableType.DROP_PARTITION);
TAlterTableDropPartitionParams addPartParams = new TAlterTableDropPartitionParams();
- addPartParams.setPartition_spec(partitionSpec_.toThrift());
- addPartParams.setIf_exists(ifExists_);
+ addPartParams.setPartition_set(partitionSet_.toThrift());
+ addPartParams.setIf_exists(!partitionSet_.getPartitionShouldExist());
addPartParams.setPurge(purgePartition_);
params.setDrop_partition_params(addPartParams);
return params;
@@ -72,8 +72,8 @@ public class AlterTableDropPartitionStmt extends AlterTableStmt {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
- if (!ifExists_) partitionSpec_.setPartitionShouldExist();
- partitionSpec_.setPrivilegeRequirement(Privilege.ALTER);
- partitionSpec_.analyze(analyzer);
+ if (!ifExists_) partitionSet_.setPartitionShouldExist();
+ partitionSet_.setPrivilegeRequirement(Privilege.ALTER);
+ partitionSet_.analyze(analyzer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java
index 56122e1..931c6a9 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetCachedStmt.java
@@ -17,6 +17,8 @@
package org.apache.impala.analysis;
+import java.util.List;
+
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.Table;
@@ -27,14 +29,14 @@ import org.apache.impala.thrift.TAlterTableType;
import com.google.common.base.Preconditions;
/**
- * Represents an ALTER TABLE [PARTITION partitionSpec] SET [UNCACHED|CACHED 'pool'].
+ * Represents an ALTER TABLE [PARTITION partitionSet] SET [UNCACHED|CACHED 'pool'].
*/
public class AlterTableSetCachedStmt extends AlterTableSetStmt {
private final HdfsCachingOp cacheOp_;
public AlterTableSetCachedStmt(TableName tableName,
- PartitionSpec partitionSpec, HdfsCachingOp cacheOp) {
- super(tableName, partitionSpec);
+ PartitionSet partitionSet, HdfsCachingOp cacheOp) {
+ super(tableName, partitionSet);
Preconditions.checkNotNull(cacheOp);
cacheOp_ = cacheOp;
}
@@ -45,8 +47,8 @@ public class AlterTableSetCachedStmt extends AlterTableSetStmt {
params.setAlter_type(TAlterTableType.SET_CACHED);
TAlterTableSetCachedParams cachingParams =
new TAlterTableSetCachedParams();
- if (getPartitionSpec() != null) {
- cachingParams.setPartition_spec(getPartitionSpec().toThrift());
+ if (getPartitionSet() != null) {
+ cachingParams.setPartition_set(getPartitionSet().toThrift());
}
cachingParams.setCache_op(cacheOp_.toThrift());
params.setSet_cached_params(cachingParams);
@@ -66,18 +68,18 @@ public class AlterTableSetCachedStmt extends AlterTableSetStmt {
}
if (cacheOp_.shouldCache()) {
- boolean isCacheable;
- PartitionSpec partSpec = getPartitionSpec();
+ boolean isCacheable = true;
+ PartitionSet partitionSet = getPartitionSet();
HdfsTable hdfsTable = (HdfsTable)table;
StringBuilder nameSb = new StringBuilder();
- if (partSpec != null) {
- HdfsPartition part = hdfsTable.getPartition(partSpec.getPartitionSpecKeyValues());
- if (part == null) {
- throw new AnalysisException("Partition spec does not exist: " +
- partSpec.toSql());
+ if (partitionSet != null) {
+ List<HdfsPartition> parts = partitionSet.getPartitions();
+ nameSb.append("Partition(s) (");
+ for(HdfsPartition part: parts) {
+ isCacheable = isCacheable && part.isCacheable();
+ if(!part.isCacheable()) nameSb.append(part.getPartitionName());
}
- isCacheable = part.isCacheable();
- nameSb.append("Partition (" + part.getPartitionName() + ")");
+ nameSb.append(")");
} else {
isCacheable = hdfsTable.isCacheable();
nameSb.append("Table ").append(table.getFullName());
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableSetFileFormatStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetFileFormatStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetFileFormatStmt.java
index ff8d878..5b53b80 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetFileFormatStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetFileFormatStmt.java
@@ -24,14 +24,14 @@ import org.apache.impala.thrift.TAlterTableType;
import org.apache.impala.thrift.THdfsFileFormat;
/**
- * Represents an ALTER TABLE [PARTITION partitionSpec] SET FILEFORMAT statement.
+ * Represents an ALTER TABLE [PARTITION partitionSet] SET FILEFORMAT statement.
*/
public class AlterTableSetFileFormatStmt extends AlterTableSetStmt {
private final THdfsFileFormat fileFormat_;
public AlterTableSetFileFormatStmt(TableName tableName,
- PartitionSpec partitionSpec, THdfsFileFormat fileFormat) {
- super(tableName, partitionSpec);
+ PartitionSet partitionSet, THdfsFileFormat fileFormat) {
+ super(tableName, partitionSet);
this.fileFormat_ = fileFormat;
}
@@ -43,8 +43,8 @@ public class AlterTableSetFileFormatStmt extends AlterTableSetStmt {
params.setAlter_type(TAlterTableType.SET_FILE_FORMAT);
TAlterTableSetFileFormatParams fileFormatParams =
new TAlterTableSetFileFormatParams(fileFormat_);
- if (getPartitionSpec() != null) {
- fileFormatParams.setPartition_spec(getPartitionSpec().toThrift());
+ if (getPartitionSet() != null) {
+ fileFormatParams.setPartition_set(getPartitionSet().toThrift());
}
params.setSet_file_format_params(fileFormatParams);
return params;
@@ -54,4 +54,4 @@ public class AlterTableSetFileFormatStmt extends AlterTableSetStmt {
public void analyze(Analyzer analyzer) throws AnalysisException {
super.analyze(analyzer);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableSetLocationStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetLocationStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetLocationStmt.java
index 0219b8c..d7a7448 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetLocationStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetLocationStmt.java
@@ -17,6 +17,10 @@
package org.apache.impala.analysis;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.hadoop.fs.permission.FsAction;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsTable;
@@ -25,18 +29,23 @@ import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TAlterTableParams;
import org.apache.impala.thrift.TAlterTableSetLocationParams;
import org.apache.impala.thrift.TAlterTableType;
+import org.apache.impala.thrift.TPartitionKeyValue;
+import com.google.common.base.Function;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.fs.permission.FsAction;
+import com.google.common.collect.Lists;
/**
* Represents an ALTER TABLE [PARTITION partitionSpec] SET LOCATION statement.
*/
public class AlterTableSetLocationStmt extends AlterTableSetStmt {
+ // max num of partitions printed during error reporting.
+ private static final int NUM_PARTITION_LOG_LIMIT = 3;
private final HdfsUri location_;
public AlterTableSetLocationStmt(TableName tableName,
- PartitionSpec partitionSpec, HdfsUri location) {
- super(tableName, partitionSpec);
+ PartitionSet partitionSet, HdfsUri location) {
+ super(tableName, partitionSet);
Preconditions.checkNotNull(location);
this.location_ = location;
}
@@ -49,8 +58,10 @@ public class AlterTableSetLocationStmt extends AlterTableSetStmt {
params.setAlter_type(TAlterTableType.SET_LOCATION);
TAlterTableSetLocationParams locationParams =
new TAlterTableSetLocationParams(location_.toString());
- if (getPartitionSpec() != null) {
- locationParams.setPartition_spec(getPartitionSpec().toThrift());
+ if (getPartitionSet() != null) {
+ List<List<TPartitionKeyValue>> tPartitionSet = getPartitionSet().toThrift();
+ Preconditions.checkState(tPartitionSet.size() == 1);
+ locationParams.setPartition_spec(tPartitionSet.get(0));
}
params.setSet_location_params(locationParams);
return params;
@@ -65,16 +76,31 @@ public class AlterTableSetLocationStmt extends AlterTableSetStmt {
Preconditions.checkNotNull(table);
if (table instanceof HdfsTable) {
HdfsTable hdfsTable = (HdfsTable) table;
- if (getPartitionSpec() != null) {
+ if (getPartitionSet() != null) {
// Targeting a partition rather than a table.
- PartitionSpec partitionSpec = getPartitionSpec();
- HdfsPartition partition = hdfsTable.getPartition(
- partitionSpec.getPartitionSpecKeyValues());
- Preconditions.checkNotNull(partition);
- if (partition.isMarkedCached()) {
+ List<HdfsPartition> partitions = getPartitionSet().getPartitions();
+ if (partitions.size() != 1) {
+ // Sort the partitions to get a consistent error reporting.
+ List<HdfsPartition> sortedPartitions = Lists.newArrayList(partitions);
+ Collections.sort(sortedPartitions);
+ List<String> sortedPartitionNames =
+ Lists.transform(sortedPartitions.subList(0, NUM_PARTITION_LOG_LIMIT),
+ new Function<HdfsPartition, String>() {
+ @Override
+ public String apply(HdfsPartition hdfsPartition) {
+ return hdfsPartition.getPartitionName();
+ }
+ });
+ throw new AnalysisException(String.format(
+ "Partition expr in set location statements can only match " +
+ "one partition. Too many matched partitions %s %s",
+ Joiner.on(",").join(sortedPartitionNames),
+ sortedPartitions.size() < partitions.size() ? "..." : "."));
+ }
+ if (partitions.get(0).isMarkedCached()) {
throw new AnalysisException(String.format("Target partition is cached, " +
"please uncache before changing the location using: ALTER TABLE %s %s " +
- "SET UNCACHED", table.getFullName(), partitionSpec.toSql()));
+ "SET UNCACHED", table.getFullName(), getPartitionSet().toSql()));
}
} else if (hdfsTable.isMarkedCached()) {
throw new AnalysisException(String.format("Target table is cached, please " +
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableSetStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetStmt.java
index a37b949..178d28b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetStmt.java
@@ -26,15 +26,15 @@ import org.apache.impala.common.AnalysisException;
* Base class for all ALTER TABLE ... SET statements
*/
public class AlterTableSetStmt extends AlterTableStmt {
- protected final PartitionSpec partitionSpec_;
+ protected final PartitionSet partitionSet_;
- public AlterTableSetStmt(TableName tableName, PartitionSpec partitionSpec) {
+ public AlterTableSetStmt(TableName tableName, PartitionSet partitionSet) {
super(tableName);
- partitionSpec_ = partitionSpec;
- if (partitionSpec_ != null) partitionSpec_.setTableName(tableName);
+ partitionSet_ = partitionSet;
+ if (partitionSet_ != null) partitionSet_.setTableName(tableName);
}
- public PartitionSpec getPartitionSpec() { return partitionSpec_; }
+ PartitionSet getPartitionSet() { return partitionSet_; }
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
@@ -49,10 +49,10 @@ public class AlterTableSetStmt extends AlterTableStmt {
}
// Altering the table rather than the partition.
- if (partitionSpec_ == null) return;
+ if (partitionSet_ == null) return;
- partitionSpec_.setPartitionShouldExist();
- partitionSpec_.setPrivilegeRequirement(Privilege.ALTER);
- partitionSpec_.analyze(analyzer);
+ partitionSet_.setPartitionShouldExist();
+ partitionSet_.setPrivilegeRequirement(Privilege.ALTER);
+ partitionSet_.analyze(analyzer);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
index f2a81cd..08007b3 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableSetTblProperties.java
@@ -47,9 +47,9 @@ public class AlterTableSetTblProperties extends AlterTableSetStmt {
private final TTablePropertyType targetProperty_;
private final HashMap<String, String> tblProperties_;
- public AlterTableSetTblProperties(TableName tableName, PartitionSpec partitionSpec,
+ public AlterTableSetTblProperties(TableName tableName, PartitionSet partitionSet,
TTablePropertyType targetProperty, HashMap<String, String> tblProperties) {
- super(tableName, partitionSpec);
+ super(tableName, partitionSet);
Preconditions.checkNotNull(tblProperties);
Preconditions.checkNotNull(targetProperty);
targetProperty_ = targetProperty;
@@ -67,8 +67,8 @@ public class AlterTableSetTblProperties extends AlterTableSetStmt {
new TAlterTableSetTblPropertiesParams();
tblPropertyParams.setTarget(targetProperty_);
tblPropertyParams.setProperties(tblProperties_);
- if (partitionSpec_ != null) {
- tblPropertyParams.setPartition_spec(partitionSpec_.toThrift());
+ if (partitionSet_ != null) {
+ tblPropertyParams.setPartition_set(partitionSet_.toThrift());
}
params.setSet_tbl_properties_params(tblPropertyParams);
return params;
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
index 223abe9..d86448b 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
@@ -69,23 +69,33 @@ public abstract class AlterTableStmt extends StatementBase {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- table_ = analyzer.getTable(tableName_, Privilege.ALTER);
+ // Resolve and analyze this table ref so we can evaluate partition predicates.
+ TableRef tableRef = new TableRef(tableName_.toPath(), null, Privilege.ALTER);
+ tableRef = analyzer.resolveTableRef(tableRef);
+ Preconditions.checkNotNull(tableRef);
+ tableRef.analyze(analyzer);
+ if (tableRef instanceof InlineViewRef) {
+ throw new AnalysisException(String.format(
+ "ALTER TABLE not allowed on a view: %s", tableName_));
+ }
+ if (tableRef instanceof CollectionTableRef) {
+ throw new AnalysisException(String.format(
+ "ALTER TABLE not allowed on a nested collection: %s", tableName_));
+ }
+ Preconditions.checkState(tableRef instanceof BaseTableRef);
+ table_ = tableRef.getTable();
if (table_ instanceof KuduTable
&& !(this instanceof AlterTableSetTblProperties)
&& !(this instanceof AlterTableSetColumnStats)
&& !(this instanceof AlterTableOrViewRenameStmt)) {
throw new AnalysisException(String.format(
- "ALTER TABLE not allowed on Kudu table: %s", table_.getFullName()));
- }
- if (table_ instanceof View) {
- throw new AnalysisException(String.format(
- "ALTER TABLE not allowed on a view: %s", table_.getFullName()));
+ "ALTER TABLE not allowed on Kudu table: %s", tableName_));
}
if (table_ instanceof DataSourceTable
&& !(this instanceof AlterTableSetColumnStats)) {
throw new AnalysisException(String.format(
"ALTER TABLE not allowed on a table produced by a data source: %s",
- table_.getFullName()));
+ tableName_));
}
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index 5a34fbc..0638bed 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -553,11 +553,11 @@ public class Analyzer {
if (rawPath.size() > 1) {
registerPrivReq(new PrivilegeRequestBuilder()
.onTable(rawPath.get(0), rawPath.get(1))
- .allOf(Privilege.SELECT).toRequest());
+ .allOf(tableRef.getPrivilege()).toRequest());
}
registerPrivReq(new PrivilegeRequestBuilder()
.onTable(getDefaultDb(), rawPath.get(0))
- .allOf(Privilege.SELECT).toRequest());
+ .allOf(tableRef.getPrivilege()).toRequest());
}
throw e;
} catch (TableLoadingException e) {
@@ -2568,28 +2568,28 @@ public class Analyzer {
}
/**
- * Registers a table-level SELECT privilege request and an access event for auditing
- * for the given table. The given table must be a base table or a catalog view (not
- * a local view).
+ * Registers a table-level privilege request and an access event for auditing
+ * for the given table and privilege. The table must be a base table or a
+ * catalog view (not a local view).
*/
- public void registerAuthAndAuditEvent(Table table, Analyzer analyzer) {
+ public void registerAuthAndAuditEvent(Table table, Privilege priv) {
// Add access event for auditing.
if (table instanceof View) {
View view = (View) table;
Preconditions.checkState(!view.isLocalView());
- analyzer.addAccessEvent(new TAccessEvent(
+ addAccessEvent(new TAccessEvent(
table.getFullName(), TCatalogObjectType.VIEW,
- Privilege.SELECT.toString()));
+ priv.toString()));
} else {
- analyzer.addAccessEvent(new TAccessEvent(
+ addAccessEvent(new TAccessEvent(
table.getFullName(), TCatalogObjectType.TABLE,
- Privilege.SELECT.toString()));
+ priv.toString()));
}
// Add privilege request.
TableName tableName = table.getTableName();
- analyzer.registerPrivReq(new PrivilegeRequestBuilder()
+ registerPrivReq(new PrivilegeRequestBuilder()
.onTable(tableName.getDb(), tableName.getTbl())
- .allOf(Privilege.SELECT).toRequest());
+ .allOf(priv).toRequest());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
index fd1c455..1691315 100644
--- a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
@@ -58,7 +58,7 @@ public class BaseTableRef extends TableRef {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
if (isAnalyzed_) return;
- analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer);
+ analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), priv_);
desc_ = analyzer.registerTableRef(this);
isAnalyzed_ = true;
analyzeHints(analyzer);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
index e8abaa8..05a3543 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CollectionTableRef.java
@@ -103,7 +103,7 @@ public class CollectionTableRef extends TableRef {
// Register a table-level privilege request as well as a column-level privilege request
// for the collection-typed column.
Preconditions.checkNotNull(resolvedPath_.getRootTable());
- analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), analyzer);
+ analyzer.registerAuthAndAuditEvent(resolvedPath_.getRootTable(), priv_);
analyzer.registerPrivReq(new PrivilegeRequestBuilder().
allOf(Privilege.SELECT).onColumn(desc_.getTableName().getDb(),
desc_.getTableName().getTbl(), desc_.getPath().getRawPath().get(0))
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index 5ad3cad..7aface4 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -21,8 +21,6 @@ import java.util.Iterator;
import java.util.List;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.log4j.Logger;
-
import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.Column;
import org.apache.impala.catalog.HBaseTable;
@@ -30,13 +28,14 @@ import org.apache.impala.catalog.HdfsPartition;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.Type;
-import org.apache.impala.catalog.View;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.PrintUtils;
import org.apache.impala.service.BackendConfig;
import org.apache.impala.thrift.TComputeStatsParams;
import org.apache.impala.thrift.TPartitionStats;
import org.apache.impala.thrift.TTableName;
+import org.apache.log4j.Logger;
+
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -99,9 +98,9 @@ public class ComputeStatsStmt extends StatementBase {
// PARTITION_ID() builtin)
private final List<List<String>> expectedPartitions_ = Lists.newArrayList();
- // If non-null, the partition that an incremental computation might apply to. Must be
+ // If non-null, partitions that an incremental computation might apply to. Must be
// null if this is a non-incremental computation.
- private PartitionSpec partitionSpec_ = null;
+ private PartitionSet partitionSet_ = null;
// The maximum number of partitions that may be explicitly selected by filter
// predicates. Any query that selects more than this automatically drops back to a full
@@ -120,20 +119,20 @@ public class ComputeStatsStmt extends StatementBase {
/**
* Constructor for the incremental form of COMPUTE STATS. If isIncremental is true,
* statistics will be recomputed incrementally; if false they will be recomputed for the
- * whole table. The partition spec partSpec can specify a single partition whose stats
- * should be recomputed.
+ * whole table. The partition set partitionSet can specify a list of partitions whose
+ * stats should be recomputed.
*/
protected ComputeStatsStmt(TableName tableName, boolean isIncremental,
- PartitionSpec partSpec) {
+ PartitionSet partitionSet) {
Preconditions.checkState(tableName != null && !tableName.isEmpty());
- Preconditions.checkState(isIncremental || partSpec == null);
+ Preconditions.checkState(isIncremental || partitionSet == null);
this.tableName_ = tableName;
this.table_ = null;
this.isIncremental_ = isIncremental;
- this.partitionSpec_ = partSpec;
- if (partitionSpec_ != null) {
- partitionSpec_.setTableName(tableName);
- partitionSpec_.setPrivilegeRequirement(Privilege.ALTER);
+ this.partitionSet_ = partitionSet;
+ if (partitionSet_ != null) {
+ partitionSet_.setTableName(tableName);
+ partitionSet_.setPrivilegeRequirement(Privilege.ALTER);
}
}
@@ -245,17 +244,25 @@ public class ComputeStatsStmt extends StatementBase {
*/
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- table_ = analyzer.getTable(tableName_, Privilege.ALTER);
- String sqlTableName = table_.getTableName().toSql();
- if (table_ instanceof View) {
+ // Resolve and analyze this table ref so we can evaluate partition predicates.
+ TableRef tableRef = new TableRef(tableName_.toPath(), null, Privilege.ALTER);
+ tableRef = analyzer.resolveTableRef(tableRef);
+ Preconditions.checkNotNull(tableRef);
+ tableRef.analyze(analyzer);
+ if (tableRef instanceof InlineViewRef) {
+ throw new AnalysisException(String.format(
+ "COMPUTE STATS not supported for view: %s", tableName_));
+ }
+ if (tableRef instanceof CollectionTableRef) {
throw new AnalysisException(String.format(
- "COMPUTE STATS not supported for view %s", sqlTableName));
+ "COMPUTE STATS not supported for nested collection: %s", tableName_));
}
+ table_ = analyzer.getTable(tableName_, Privilege.ALTER);
if (!(table_ instanceof HdfsTable)) {
- if (partitionSpec_ != null) {
+ if (partitionSet_ != null) {
throw new AnalysisException("COMPUTE INCREMENTAL ... PARTITION not supported " +
- "for non-HDFS table " + table_.getTableName());
+ "for non-HDFS table " + tableName_);
}
isIncremental_ = false;
}
@@ -267,22 +274,14 @@ public class ComputeStatsStmt extends StatementBase {
if (table_ instanceof HdfsTable) {
hdfsTable = (HdfsTable)table_;
if (isIncremental_ && hdfsTable.getNumClusteringCols() == 0 &&
- partitionSpec_ != null) {
- throw new AnalysisException(String.format(
- "Can't compute PARTITION stats on an unpartitioned table: %s",
- sqlTableName));
- } else if (partitionSpec_ != null) {
- partitionSpec_.setPartitionShouldExist();
- partitionSpec_.analyze(analyzer);
- for (PartitionKeyValue kv: partitionSpec_.getPartitionSpecKeyValues()) {
- // TODO: We could match the dynamic keys (i.e. as wildcards) as well, but that
- // would involve looping over all partitions and seeing which match the
- // partition spec.
- if (!kv.isStatic()) {
- throw new AnalysisException("All partition keys must have values: " +
- kv.toString());
- }
- }
+ partitionSet_ != null) {
+ throw new AnalysisException(String.format(
+ "Can't compute PARTITION stats on an unpartitioned table: %s",
+ tableName_));
+ } else if (partitionSet_ != null) {
+ Preconditions.checkState(tableRef instanceof BaseTableRef);
+ partitionSet_.setPartitionShouldExist();
+ partitionSet_.analyze(analyzer);
}
// For incremental stats, estimate the size of intermediate stats and report an
// error if the estimate is greater than --inc_stats_size_limit_bytes in bytes
@@ -305,7 +304,7 @@ public class ComputeStatsStmt extends StatementBase {
// incremental computation.
List<String> filterPreds = Lists.newArrayList();
if (isIncremental_) {
- if (partitionSpec_ == null) {
+ if (partitionSet_ == null) {
// If any column does not have stats, we recompute statistics for all partitions
// TODO: need a better way to invalidate stats for all partitions, so that we can
// use this logic to only recompute new / changed columns.
@@ -355,23 +354,23 @@ public class ComputeStatsStmt extends StatementBase {
expectAllPartitions_ = true;
}
} else {
- // Always compute stats on a particular partition when told to.
+ List<HdfsPartition> targetPartitions = partitionSet_.getPartitions();
+
+ // Always compute stats on a set of partitions when told to.
List<String> partitionConjuncts = Lists.newArrayList();
- for (PartitionKeyValue kv: partitionSpec_.getPartitionSpecKeyValues()) {
- partitionConjuncts.add(kv.toPredicateSql());
+ for (HdfsPartition targetPartition : targetPartitions) {
+ partitionConjuncts.add(targetPartition.getConjunctSql());
+ List<String> partValues = Lists.newArrayList();
+ for (LiteralExpr partValue: targetPartition.getPartitionValues()) {
+ partValues.add(PartitionKeyValue.getPartitionKeyValueString(partValue,
+ "NULL"));
+ }
+ expectedPartitions_.add(partValues);
}
filterPreds.add("(" + Joiner.on(" AND ").join(partitionConjuncts) + ")");
- HdfsPartition targetPartition =
- hdfsTable.getPartition(partitionSpec_.getPartitionSpecKeyValues());
- List<String> partValues = Lists.newArrayList();
- for (LiteralExpr partValue: targetPartition.getPartitionValues()) {
- partValues.add(PartitionKeyValue.getPartitionKeyValueString(partValue,
- "NULL"));
- }
- expectedPartitions_.add(partValues);
- for (HdfsPartition p: hdfsTable.getPartitions()) {
+ for (HdfsPartition p : hdfsTable.getPartitions()) {
if (p.isDefaultPartition()) continue;
- if (p == targetPartition) continue;
+ if (targetPartitions.contains(p)) continue;
TPartitionStats partStats = p.getPartitionStats();
if (partStats != null) validPartStats_.add(partStats);
}
@@ -408,7 +407,7 @@ public class ComputeStatsStmt extends StatementBase {
tableStatsSelectList.addAll(partitionColsSelectList);
tableStatsQueryBuilder.append(Joiner.on(", ").join(tableStatsSelectList));
- tableStatsQueryBuilder.append(" FROM " + sqlTableName);
+ tableStatsQueryBuilder.append(" FROM " + tableName_.toSql());
// Query for getting the per-column NDVs and number of NULLs.
List<String> columnStatsSelectList = getBaseColumnStatsQuerySelectList(analyzer);
@@ -417,7 +416,7 @@ public class ComputeStatsStmt extends StatementBase {
StringBuilder columnStatsQueryBuilder = new StringBuilder("SELECT ");
columnStatsQueryBuilder.append(Joiner.on(", ").join(columnStatsSelectList));
- columnStatsQueryBuilder.append(" FROM " + sqlTableName);
+ columnStatsQueryBuilder.append(" FROM " + tableName_.toSql());
// Add the WHERE clause to filter out partitions that we don't want to compute
// incremental stats for. While this is a win in most situations, we would like to
@@ -426,7 +425,7 @@ public class ComputeStatsStmt extends StatementBase {
// selected in) and there is no partition spec (so no single partition was explicitly
// selected in).
if (filterPreds.size() > 0 &&
- (validPartStats_.size() > 0 || partitionSpec_ != null)) {
+ (validPartStats_.size() > 0 || partitionSet_ != null)) {
String filterClause = " WHERE " + Joiner.on(" OR ").join(filterPreds);
columnStatsQueryBuilder.append(filterClause);
tableStatsQueryBuilder.append(filterClause);
@@ -529,7 +528,7 @@ public class ComputeStatsStmt extends StatementBase {
return "COMPUTE STATS " + tableName_.toSql();
} else {
return "COMPUTE INCREMENTAL STATS " + tableName_.toSql() +
- partitionSpec_ == null ? "" : partitionSpec_.toSql();
+ partitionSet_ == null ? "" : partitionSet_.toSql();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
index fe2ff84..e39071f 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DropStatsStmt.java
@@ -18,10 +18,10 @@
package org.apache.impala.analysis;
import org.apache.impala.authorization.Privilege;
-import org.apache.impala.catalog.Table;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TDropStatsParams;
import org.apache.impala.thrift.TTableName;
+
import com.google.common.base.Preconditions;
/**
@@ -31,28 +31,29 @@ import com.google.common.base.Preconditions;
public class DropStatsStmt extends StatementBase {
protected final TableName tableName_;
- // If non-null, only drop the statistics for a given partition
- PartitionSpec partitionSpec_ = null;
-
// Set during analysis
- protected String dbName_;
+ protected TableRef tableRef_;
+
+ // If non-null, only drop the statistics for a given partition
+ private final PartitionSet partitionSet_;
/**
* Constructor for building the DROP TABLE/VIEW statement
*/
public DropStatsStmt(TableName tableName) {
this.tableName_ = tableName;
+ this.partitionSet_ = null;
}
- public DropStatsStmt(TableName tableName, PartitionSpec partSpec) {
+ public DropStatsStmt(TableName tableName, PartitionSet partitionSet) {
this.tableName_ = tableName;
- this.partitionSpec_ = partSpec;
+ this.partitionSet_ = partitionSet;
}
@Override
public String toSql() {
StringBuilder sb = new StringBuilder("DROP ");
- if (partitionSpec_ == null) {
+ if (partitionSet_ == null) {
sb.append(" STATS ");
if (tableName_.getDb() != null) sb.append(tableName_.getDb() + ".");
sb.append(tableName_.toSql());
@@ -60,7 +61,7 @@ public class DropStatsStmt extends StatementBase {
sb.append(" INCREMENTAL STATS ");
if (tableName_.getDb() != null) sb.append(tableName_.getDb() + ".");
sb.append(tableName_.toSql());
- sb.append(partitionSpec_.toSql());
+ sb.append(partitionSet_.toSql());
}
return sb.toString();
}
@@ -68,9 +69,8 @@ public class DropStatsStmt extends StatementBase {
public TDropStatsParams toThrift() {
TDropStatsParams params = new TDropStatsParams();
params.setTable_name(new TTableName(getDb(), getTbl()));
-
- if (partitionSpec_ != null) {
- params.setPartition_spec(partitionSpec_.toThrift());
+ if (partitionSet_ != null) {
+ params.setPartition_set(partitionSet_.toThrift());
}
return params;
}
@@ -81,14 +81,25 @@ public class DropStatsStmt extends StatementBase {
*/
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- dbName_ = analyzer.getTargetDbName(tableName_);
- Table table = analyzer.getTable(tableName_, Privilege.ALTER);
- Preconditions.checkNotNull(table);
- if (partitionSpec_ != null) {
- partitionSpec_.setTableName(tableName_);
- partitionSpec_.setPrivilegeRequirement(Privilege.ALTER);
- partitionSpec_.setPartitionShouldExist();
- partitionSpec_.analyze(analyzer);
+ // Resolve and analyze table ref to register privilege and audit events
+ // and to allow us to evaluate partition predicates.
+ tableRef_ = new TableRef(tableName_.toPath(), null, Privilege.ALTER);
+ tableRef_ = analyzer.resolveTableRef(tableRef_);
+ Preconditions.checkNotNull(tableRef_);
+ if (tableRef_ instanceof InlineViewRef) {
+ throw new AnalysisException(
+ String.format("DROP STATS not allowed on a view: %s", tableName_));
+ }
+ if (tableRef_ instanceof CollectionTableRef) {
+ throw new AnalysisException(
+ String.format("DROP STATS not allowed on a nested collection: %s", tableName_));
+ }
+ tableRef_.analyze(analyzer);
+ if (partitionSet_ != null) {
+ partitionSet_.setTableName(tableRef_.getTable().getTableName());
+ partitionSet_.setPrivilegeRequirement(Privilege.ALTER);
+ partitionSet_.setPartitionShouldExist();
+ partitionSet_.analyze(analyzer);
}
}
@@ -97,8 +108,8 @@ public class DropStatsStmt extends StatementBase {
* the target drop table resides in.
*/
public String getDb() {
- Preconditions.checkNotNull(dbName_);
- return dbName_;
+ Preconditions.checkNotNull(tableRef_);
+ return tableRef_.getTable().getDb().getName();
}
public String getTbl() { return tableName_.getTbl(); }
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
index 3ee2070..42f9400 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InlineViewRef.java
@@ -93,7 +93,8 @@ public class InlineViewRef extends TableRef {
* C'tor for creating inline views that replace a local or catalog view ref.
*/
public InlineViewRef(View view, TableRef origTblRef) {
- super(view.getTableName().toPath(), origTblRef.getExplicitAlias());
+ super(view.getTableName().toPath(), origTblRef.getExplicitAlias(),
+ origTblRef.getPrivilege());
queryStmt_ = view.getQueryStmt().clone();
queryStmt_.reset();
if (view.isLocalView()) queryStmt_.reset();
@@ -142,7 +143,7 @@ public class InlineViewRef extends TableRef {
// Catalog views refs require special analysis settings for authorization.
boolean isCatalogView = (view_ != null && !view_.isLocalView());
if (isCatalogView) {
- analyzer.registerAuthAndAuditEvent(view_, analyzer);
+ analyzer.registerAuthAndAuditEvent(view_, priv_);
if (inlineViewAnalyzer_.isExplain()) {
// If the user does not have privileges on the view's definition
// then we report a masked authorization error so as not to reveal
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
new file mode 100644
index 0000000..b41f2f9
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSet.java
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.analysis;
+
+import java.util.List;
+import java.util.Set;
+
+import org.apache.impala.analysis.BinaryPredicate.Operator;
+import org.apache.impala.catalog.Column;
+import org.apache.impala.catalog.HdfsPartition;
+import org.apache.impala.catalog.Table;
+import org.apache.impala.common.AnalysisException;
+import org.apache.impala.common.InternalException;
+import org.apache.impala.common.Reference;
+import org.apache.impala.planner.HdfsPartitionPruner;
+import org.apache.impala.thrift.TPartitionKeyValue;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+/**
+ * Represents a set of partitions resulting from evaluating a list of partition conjuncts
+ * against a table's partition list.
+ */
+public class PartitionSet extends PartitionSpecBase {
+ private final List<Expr> partitionExprs_;
+
+ // Result of analysis
+ private List<HdfsPartition> partitions_ = Lists.newArrayList();
+
+ public PartitionSet(List<Expr> partitionExprs) {
+ this.partitionExprs_ = ImmutableList.copyOf(partitionExprs);
+ }
+
+ public List<HdfsPartition> getPartitions() { return partitions_; }
+
+ @Override
+ public void analyze(Analyzer analyzer) throws AnalysisException {
+ super.analyze(analyzer);
+ List<Expr> conjuncts = Lists.newArrayList();
+ // Do not register column-authorization requests.
+ analyzer.setEnablePrivChecks(false);
+ for (Expr e: partitionExprs_) {
+ e.analyze(analyzer);
+ e.checkReturnsBool("Partition expr", false);
+ conjuncts.addAll(e.getConjuncts());
+ }
+
+ TupleDescriptor desc = analyzer.getDescriptor(tableName_.toString());
+ List<SlotId> partitionSlots = desc.getPartitionSlots();
+ for (Expr e: conjuncts) {
+ e.foldConstantChildren(analyzer);
+ // Make sure there are no constant predicates in the partition exprs.
+ if (e.isConstant()) {
+ throw new AnalysisException(String.format("Invalid partition expr %s. A " +
+ "partition spec may not contain constant predicates.", e.toSql()));
+ }
+
+ // Make sure every conjunct only contains partition slot refs.
+ if(!e.isBoundBySlotIds(partitionSlots)) {
+ throw new AnalysisException("Partition exprs cannot contain non-partition " +
+ "column(s): " + e.toSql() + ".");
+ }
+ }
+
+ List<Expr> transformedConjuncts = transformPartitionConjuncts(analyzer, conjuncts);
+ addIfExists(analyzer, table_, transformedConjuncts);
+
+ try {
+ HdfsPartitionPruner pruner = new HdfsPartitionPruner(desc);
+ partitions_ = pruner.prunePartitions(analyzer, transformedConjuncts, true);
+ } catch (InternalException e) {
+ throw new AnalysisException("Partition expr evaluation failed in the backend.", e);
+ }
+
+ if (partitionShouldExist_ != null) {
+ if (partitionShouldExist_ && partitions_.isEmpty()) {
+ throw new AnalysisException("No matching partition(s) found.");
+ } else if (!partitionShouldExist_ && !partitions_.isEmpty()) {
+ throw new AnalysisException("Partition already exists.");
+ }
+ }
+ analyzer.setEnablePrivChecks(true);
+ }
+
+ // Check if we should add IF EXISTS. Fully-specified partition specs don't add it for
+ // backwards compatibility, while more general partition expressions or partially
+ // specified partition specs add IF EXISTS by setting partitionShouldExists_ to null.
+ // The given conjuncts are assumed to only reference partition columns.
+ private void addIfExists(
+ Analyzer analyzer, Table table, List<Expr> conjuncts) {
+ boolean add = false;
+ Set<String> partColNames = Sets.newHashSet();
+ Reference<SlotRef> slotRef = new Reference<>();
+ for (Expr e : conjuncts) {
+ if (e instanceof BinaryPredicate) {
+ BinaryPredicate bp = (BinaryPredicate) e;
+ if (bp.getOp() != Operator.EQ || !bp.isSingleColumnPredicate(slotRef, null)) {
+ add = true;
+ break;
+ }
+ Column partColumn = slotRef.getRef().getDesc().getColumn();
+ Preconditions.checkState(table.isClusteringColumn(partColumn));
+ partColNames.add(partColumn.getName());
+ } else if (e instanceof IsNullPredicate) {
+ IsNullPredicate nullPredicate = (IsNullPredicate) e;
+ Column partColumn = nullPredicate.getBoundSlot().getDesc().getColumn();
+ Preconditions.checkState(table.isClusteringColumn(partColumn));
+ partColNames.add(partColumn.getName());
+ } else {
+ add = true;
+ break;
+ }
+ }
+
+ if (add || partColNames.size() < table.getNumClusteringCols()) {
+ partitionShouldExist_ = null;
+ }
+ }
+
+ // Transform <COL> = NULL into IsNull expr; <String COL> = '' into IsNull expr and
+ // <String COL> = 'String Value' into lower case.
+ // The reason is that COL = NULL is allowed for selecting the NULL
+ // partition, but a COL = NULL predicate can never be true, so we
+ // need to transform such predicates before feeding them into the
+ // partition pruner. Same logic goes to String transformation.
+ private List<Expr> transformPartitionConjuncts(Analyzer analyzer, List<Expr> conjuncts)
+ throws AnalysisException {
+ List<Expr> transformedConjuncts = Lists.newArrayList();
+ for (Expr e : conjuncts) {
+ Expr result = e;
+ if (e instanceof BinaryPredicate) {
+ BinaryPredicate bp = ((BinaryPredicate) e);
+ if (bp.getOp() == Operator.EQ) {
+ SlotRef leftChild =
+ bp.getChild(0) instanceof SlotRef ? ((SlotRef) bp.getChild(0)) : null;
+ NullLiteral nullChild = bp.getChild(1) instanceof NullLiteral ?
+ ((NullLiteral) bp.getChild(1)) : null;
+ StringLiteral stringChild = bp.getChild(1) instanceof StringLiteral ?
+ ((StringLiteral) bp.getChild(1)) : null;
+ if (leftChild != null && nullChild != null) {
+ result = new IsNullPredicate(leftChild, false);
+ } else if (leftChild != null && stringChild != null) {
+ if (stringChild.getStringValue().isEmpty()) {
+ result = new IsNullPredicate(leftChild, false);
+ } else {
+ stringChild = new StringLiteral(stringChild.getStringValue().toLowerCase());
+ result.setChild(1, stringChild);
+ }
+ }
+ }
+ }
+ result.analyze(analyzer);
+ transformedConjuncts.add(result);
+ }
+ return transformedConjuncts;
+ }
+
+ public List<List<TPartitionKeyValue>> toThrift() {
+ List<List<TPartitionKeyValue>> thriftPartitionSet = Lists.newArrayList();
+ for (HdfsPartition hdfsPartition : partitions_) {
+ List<TPartitionKeyValue> thriftPartitionSpec = Lists.newArrayList();
+ for (int i = 0; i < table_.getNumClusteringCols(); ++i) {
+ String key = table_.getColumns().get(i).getName();
+ String value = PartitionKeyValue.getPartitionKeyValueString(
+ hdfsPartition.getPartitionValue(i), nullPartitionKeyValue_);
+ thriftPartitionSpec.add(new TPartitionKeyValue(key, value));
+ }
+ thriftPartitionSet.add(thriftPartitionSpec);
+ }
+ return thriftPartitionSet;
+ }
+
+ @Override
+ public String toSql() {
+ List<String> partitionExprStr = Lists.newArrayList();
+ for (Expr e : partitionExprs_) {
+ partitionExprStr.add(e.toSql());
+ }
+ return String.format("PARTITION (%s)", Joiner.on(", ").join(partitionExprStr));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
index 39cc1bb..b89e586 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSpec.java
@@ -38,20 +38,13 @@ import com.google.common.collect.Sets;
/*
* Represents a partition spec - a collection of partition key/values.
*/
-public class PartitionSpec implements ParseNode {
+public class PartitionSpec extends PartitionSpecBase {
private final ImmutableList<PartitionKeyValue> partitionSpec_;
- private TableName tableName_;
- private Boolean partitionShouldExist_;
- private Privilege privilegeRequirement_;
// Flag to determine if the partition already exists in the target table.
// Set during analysis.
private Boolean partitionExists_;
- // The value Hive is configured to use for NULL partition key values.
- // Set during analysis.
- private String nullPartitionKeyValue_;
-
public PartitionSpec(List<PartitionKeyValue> partitionSpec) {
this.partitionSpec_ = ImmutableList.copyOf(partitionSpec);
}
@@ -60,48 +53,14 @@ public class PartitionSpec implements ParseNode {
return partitionSpec_;
}
- public String getTbl() { return tableName_.getTbl(); }
- public void setTableName(TableName tableName) { this.tableName_ = tableName; }
public boolean partitionExists() {
Preconditions.checkNotNull(partitionExists_);
return partitionExists_;
}
- // The value Hive is configured to use for NULL partition key values.
- // Set during analysis.
- public String getNullPartitionKeyValue() {
- Preconditions.checkNotNull(nullPartitionKeyValue_);
- return nullPartitionKeyValue_;
- }
-
- // If set, an additional analysis check will be performed to validate the target table
- // contains the given partition spec.
- public void setPartitionShouldExist() { partitionShouldExist_ = Boolean.TRUE; }
-
- // If set, an additional analysis check will be performed to validate the target table
- // does not contain the given partition spec.
- public void setPartitionShouldNotExist() { partitionShouldExist_ = Boolean.FALSE; }
-
- // Set the privilege requirement for this partition spec. Must be set prior to
- // analysis.
- public void setPrivilegeRequirement(Privilege privilege) {
- privilegeRequirement_ = privilege;
- }
-
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- Preconditions.checkNotNull(tableName_);
- Preconditions.checkNotNull(privilegeRequirement_);
-
- // Skip adding an audit event when analyzing partitions. The parent table should
- // be audited outside of the PartitionSpec.
- Table table = analyzer.getTable(tableName_, privilegeRequirement_, false);
- String tableName = table.getDb().getName() + "." + getTbl();
-
- // Make sure the target table is partitioned.
- if (table.getMetaStoreTable().getPartitionKeysSize() == 0) {
- throw new AnalysisException("Table is not partitioned: " + tableName);
- }
+ super.analyze(analyzer);
// Make sure static partition key values only contain constant exprs.
for (PartitionKeyValue kv: partitionSpec_) {
@@ -110,7 +69,7 @@ public class PartitionSpec implements ParseNode {
// Get all keys in the target table.
Set<String> targetPartitionKeys = Sets.newHashSet();
- for (FieldSchema fs: table.getMetaStoreTable().getPartitionKeys()) {
+ for (FieldSchema fs: table_.getMetaStoreTable().getPartitionKeys()) {
targetPartitionKeys.add(fs.getName().toLowerCase());
}
@@ -118,7 +77,7 @@ public class PartitionSpec implements ParseNode {
if (targetPartitionKeys.size() != partitionSpec_.size()) {
throw new AnalysisException(String.format("Items in partition spec must exactly " +
"match the partition columns in the table definition: %s (%d vs %d)",
- tableName, partitionSpec_.size(), targetPartitionKeys.size()));
+ tableName_, partitionSpec_.size(), targetPartitionKeys.size()));
}
Set<String> keyNames = Sets.newHashSet();
@@ -130,14 +89,14 @@ public class PartitionSpec implements ParseNode {
throw new AnalysisException("Duplicate partition key name: " + pk.getColName());
}
- Column c = table.getColumn(pk.getColName());
+ Column c = table_.getColumn(pk.getColName());
if (c == null) {
throw new AnalysisException(String.format(
- "Partition column '%s' not found in table: %s", pk.getColName(), tableName));
+ "Partition column '%s' not found in table: %s", pk.getColName(), tableName_));
} else if (!targetPartitionKeys.contains(pk.getColName().toLowerCase())) {
throw new AnalysisException(String.format(
"Column '%s' is not a partition column in table: %s",
- pk.getColName(), tableName));
+ pk.getColName(), tableName_));
} else if (pk.getValue() instanceof NullLiteral) {
// No need for further analysis checks of this partition key value.
continue;
@@ -160,12 +119,7 @@ public class PartitionSpec implements ParseNode {
pk.getValue().toSql(), colType.toString(), pk.getColName()));
}
}
- // Only HDFS tables are partitioned.
- Preconditions.checkState(table instanceof HdfsTable);
- HdfsTable hdfsTable = (HdfsTable) table;
- nullPartitionKeyValue_ = hdfsTable.getNullPartitionKeyValue();
-
- partitionExists_ = hdfsTable.getPartition(partitionSpec_) != null;
+ partitionExists_ = table_.getPartition(partitionSpec_) != null;
if (partitionShouldExist_ != null) {
if (partitionShouldExist_ && !partitionExists_) {
throw new AnalysisException("Partition spec does not exist: (" +
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java b/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
new file mode 100644
index 0000000..7d9a800
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/analysis/PartitionSpecBase.java
@@ -0,0 +1,94 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.package com.cloudera.impala.analysis;
+
+package org.apache.impala.analysis;
+
+import org.apache.impala.authorization.Privilege;
+import org.apache.impala.catalog.Table;
+import org.apache.impala.catalog.HdfsTable;
+import org.apache.impala.common.AnalysisException;
+import com.google.common.base.Preconditions;
+
+/**
+ * Base class for PartitionSpec and PartitionSet containing the partition
+ * specifications of related DDL operations.
+ */
+public abstract class PartitionSpecBase implements ParseNode {
+ protected HdfsTable table_;
+ protected TableName tableName_;
+ protected Boolean partitionShouldExist_;
+ protected Privilege privilegeRequirement_;
+ // The value Hive is configured to use for NULL partition key values.
+ // Set during analysis.
+ protected String nullPartitionKeyValue_;
+
+ public String getTbl() { return tableName_.getTbl(); }
+
+ public void setTableName(TableName tableName) {this.tableName_ = tableName; }
+
+ // The value Hive is configured to use for NULL partition key values.
+ // Set during analysis.
+ public String getNullPartitionKeyValue() {
+ Preconditions.checkNotNull(nullPartitionKeyValue_);
+ return nullPartitionKeyValue_;
+ }
+
+ // If set, an additional analysis check will be performed to validate the target table
+ // contains the given partition spec.
+ public void setPartitionShouldExist() { partitionShouldExist_ = Boolean.TRUE; }
+
+ // If set, an additional analysis check will be performed to validate the target table
+ // does not contain the given partition spec.
+ public void setPartitionShouldNotExist() { partitionShouldExist_ = Boolean.FALSE; }
+
+ public boolean getPartitionShouldExist() {
+ return partitionShouldExist_ != null && partitionShouldExist_;
+ }
+
+ public boolean getPartitionShouldNotExist() {
+ return partitionShouldExist_ != null && !partitionShouldExist_;
+ }
+
+ // Set the privilege requirement for this partition spec. Must be set prior to
+ // analysis.
+ public void setPrivilegeRequirement(Privilege privilege) {
+ privilegeRequirement_ = privilege;
+ }
+
+ @Override
+ public void analyze(Analyzer analyzer) throws AnalysisException {
+ Preconditions.checkNotNull(tableName_);
+ Preconditions.checkNotNull(privilegeRequirement_);
+
+ // Skip adding an audit event when analyzing partitions. The parent table should
+ // be audited outside of the PartitionSpec.
+ Table table = analyzer.getTable(tableName_, privilegeRequirement_, false);
+
+ // Make sure the target table is partitioned.
+ if (table.getMetaStoreTable().getPartitionKeysSize() == 0) {
+ throw new AnalysisException("Table is not partitioned: " + tableName_);
+ }
+
+ // Only HDFS tables are partitioned.
+ Preconditions.checkState(table instanceof HdfsTable);
+ table_ = (HdfsTable) table;
+ nullPartitionKeyValue_ = table_.getNullPartitionKeyValue();
+ }
+
+ @Override
+ public abstract String toSql();
+}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java b/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
index bec1ed3..d102bb9 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowFilesStmt.java
@@ -27,8 +27,6 @@ import org.apache.impala.catalog.Table;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.thrift.TTableName;
import com.google.common.base.Preconditions;
-import com.google.common.base.Joiner;
-import com.google.common.collect.Lists;
/**
* Representation of a SHOW FILES statement.
@@ -38,52 +36,60 @@ import com.google.common.collect.Lists;
*
*/
public class ShowFilesStmt extends StatementBase {
- private TableName tableName_;
+ private final TableName tableName_;
// Show files for all the partitions if this is null.
- private final PartitionSpec partitionSpec_;
+ private final PartitionSet partitionSet_;
// Set during analysis.
protected Table table_;
- public ShowFilesStmt(TableName tableName, PartitionSpec partitionSpec) {
+ public ShowFilesStmt(TableName tableName, PartitionSet partitionSet) {
this.tableName_ = tableName;
- this.partitionSpec_ = partitionSpec;
+ this.partitionSet_ = partitionSet;
}
@Override
public String toSql() {
StringBuilder strBuilder = new StringBuilder();
strBuilder.append("SHOW FILES IN " + tableName_.toString());
- if (partitionSpec_ != null) strBuilder.append(" " + partitionSpec_.toSql());
+ if (partitionSet_ != null) strBuilder.append(" " + partitionSet_.toSql());
return strBuilder.toString();
}
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- if (!tableName_.isFullyQualified()) {
- tableName_ = new TableName(analyzer.getDefaultDb(), tableName_.getTbl());
+ // Resolve and analyze table ref to register privilege and audit events
+ // and to allow us to evaluate partition predicates.
+ TableRef tableRef = new TableRef(tableName_.toPath(), null, Privilege.VIEW_METADATA);
+ tableRef = analyzer.resolveTableRef(tableRef);
+ if (tableRef instanceof InlineViewRef ||
+ tableRef instanceof CollectionTableRef) {
+ throw new AnalysisException(String.format(
+ "SHOW FILES not applicable to a non hdfs table: %s", tableName_));
}
- table_ = analyzer.getTable(tableName_, Privilege.VIEW_METADATA);
+ table_ = tableRef.getTable();
+ Preconditions.checkNotNull(table_);
if (!(table_ instanceof HdfsTable)) {
throw new AnalysisException(String.format(
- "SHOW FILES not applicable to a non hdfs table: %s", table_.getFullName()));
+ "SHOW FILES not applicable to a non hdfs table: %s", tableName_));
}
+ tableRef.analyze(analyzer);
// Analyze the partition spec, if one was specified.
- if (partitionSpec_ != null) {
- partitionSpec_.setTableName(tableName_);
- partitionSpec_.setPartitionShouldExist();
- partitionSpec_.setPrivilegeRequirement(Privilege.VIEW_METADATA);
- partitionSpec_.analyze(analyzer);
+ if (partitionSet_ != null) {
+ partitionSet_.setTableName(table_.getTableName());
+ partitionSet_.setPartitionShouldExist();
+ partitionSet_.setPrivilegeRequirement(Privilege.VIEW_METADATA);
+ partitionSet_.analyze(analyzer);
}
}
public TShowFilesParams toThrift() {
TShowFilesParams params = new TShowFilesParams();
- params.setTable_name(new TTableName(tableName_.getDb(), tableName_.getTbl()));
- if (partitionSpec_ != null) {
- params.setPartition_spec(partitionSpec_.toThrift());
+ params.setTable_name(new TTableName(table_.getDb().getName(), table_.getName()));
+ if (partitionSet_ != null) {
+ params.setPartition_set(partitionSet_.toThrift());
}
return params;
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/fe/src/main/java/org/apache/impala/analysis/TableRef.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableRef.java b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
index 6403091..d6bbfd2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableRef.java
@@ -22,6 +22,7 @@ import java.util.Collections;
import java.util.List;
import java.util.Set;
+import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.Table;
import org.apache.impala.common.AnalysisException;
@@ -77,6 +78,9 @@ public class TableRef implements ParseNode {
// Indicates whether this table ref is given an explicit alias,
protected boolean hasExplicitAlias_;
+ // Analysis registers privilege and/or audit requests based on this privilege.
+ protected final Privilege priv_;
+
protected JoinOperator joinOp_;
protected ArrayList<String> joinHints_;
protected List<String> usingColNames_;
@@ -125,7 +129,10 @@ public class TableRef implements ParseNode {
/////////////////////////////////////////
public TableRef(List<String> path, String alias) {
- super();
+ this(path, alias, Privilege.SELECT);
+ }
+
+ public TableRef(List<String> path, String alias, Privilege priv) {
rawPath_ = path;
if (alias != null) {
aliases_ = new String[] { alias.toLowerCase() };
@@ -133,6 +140,7 @@ public class TableRef implements ParseNode {
} else {
hasExplicitAlias_ = false;
}
+ priv_ = priv;
isAnalyzed_ = false;
replicaPreference_ = null;
randomReplica_ = false;
@@ -146,6 +154,7 @@ public class TableRef implements ParseNode {
resolvedPath_ = other.resolvedPath_;
aliases_ = other.aliases_;
hasExplicitAlias_ = other.hasExplicitAlias_;
+ priv_ = other.priv_;
joinOp_ = other.joinOp_;
joinHints_ =
(other.joinHints_ != null) ? Lists.newArrayList(other.joinHints_) : null;
@@ -252,6 +261,7 @@ public class TableRef implements ParseNode {
Preconditions.checkNotNull(resolvedPath_);
return resolvedPath_.getRootTable();
}
+ public Privilege getPrivilege() { return priv_; }
public ArrayList<String> getJoinHints() { return joinHints_; }
public ArrayList<String> getTableHints() { return tableHints_; }
public Expr getOnClause() { return onClause_; }
[6/9] incubator-impala git commit: IMPALA-4433: Always generate
testdata using the same time zone setting
Posted by sa...@apache.org.
IMPALA-4433: Always generate testdata using the same time zone setting
Before this change, testdata was generated using the
java.util.TimeZone.getDefault() TimeZone of the machine it was running
on. This patch standardizes on "America/Los_Angeles", which matches
the existing expected results in the end-to-end tests.
Change-Id: Iaf7cc796e44e9ff64880f9ae852f40961592f279
Reviewed-on: http://gerrit.cloudera.org:8080/5058
Reviewed-by: Jim Apple <jb...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0ea4a666
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0ea4a666
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0ea4a666
Branch: refs/heads/master
Commit: 0ea4a666dc12e093cc12ca4cd640f0ede51661a2
Parents: f4a5d86
Author: Jim Apple <jb...@cloudera.com>
Authored: Fri Nov 11 12:05:30 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 04:18:33 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/datagenerator/TestDataGenerator.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0ea4a666/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
----------------------------------------------------------------------
diff --git a/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java b/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
index 7753ac0..29fd0d7 100644
--- a/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
+++ b/testdata/src/main/java/org/apache/impala/datagenerator/TestDataGenerator.java
@@ -25,6 +25,7 @@ import java.math.BigDecimal;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.GregorianCalendar;
+import java.util.TimeZone;
class TestDataGenerator {
// 2 years
@@ -152,6 +153,10 @@ class TestDataGenerator {
System.err.println("Usage: " + "TestDataGenerator BaseOutputDirectory");
}
+ // The TimeZone should be the same no matter what the TimeZone is of the computer
+ // running this code, in order to ensure the generated data is always the same.
+ TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"));
+
// Generate AllTypes
String dirName = args[0] + "/AllTypes";
File dir = new File(dirName);
[8/9] incubator-impala git commit: IMPALA-4479: Use correct isSet()
thrift function when evaluating constant bool exprs.
Posted by sa...@apache.org.
IMPALA-4479: Use correct isSet() thrift function when evaluating constant bool exprs.
Change-Id: Ie3ba195a5241ca630bd0cf71b83d423733b06546
Reviewed-on: http://gerrit.cloudera.org:8080/5088
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/91b5264e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/91b5264e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/91b5264e
Branch: refs/heads/master
Commit: 91b5264e5239b2cf3ad4ea896eb28dd5fbdcb74a
Parents: e6e2bae
Author: Alex Behm <al...@cloudera.com>
Authored: Mon Nov 14 15:44:10 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 11:17:43 2016 +0000
----------------------------------------------------------------------
.../main/java/org/apache/impala/analysis/LiteralExpr.java | 2 +-
.../functional-planner/queries/PlannerTest/kudu.test | 9 +++++++++
2 files changed, 10 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/91b5264e/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java b/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
index 84368f5..de73770 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LiteralExpr.java
@@ -174,7 +174,7 @@ public abstract class LiteralExpr extends Expr implements Comparable<LiteralExpr
result = new NullLiteral();
break;
case BOOLEAN:
- if (val.isBool_val()) result = new BoolLiteral(val.bool_val);
+ if (val.isSetBool_val()) result = new BoolLiteral(val.bool_val);
break;
case TINYINT:
if (val.isSetByte_val()) {
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/91b5264e/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test b/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
index fbb7f7f..6f7bbff 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/kudu.test
@@ -272,3 +272,12 @@ INSERT INTO KUDU [functional_kudu.testtbl]
|
00:SCAN KUDU [functional_kudu.testtbl]
====
+# IMPALA-4479: Test proper folding of constant boolean exprs.
+select * from functional_kudu.alltypes
+where bool_col = (true and false)
+---- PLAN
+PLAN-ROOT SINK
+|
+00:SCAN KUDU [functional_kudu.alltypes]
+ kudu predicates: bool_col = FALSE
+====
[2/9] incubator-impala git commit: IMPALA-1654: General partition
exprs in DDL operations.
Posted by sa...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
index 3b1d86b..3230b9e 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/alter-table.test
@@ -247,6 +247,9 @@ bigint,bigint,string,string,string,string,string,string
# change the table fileformat to match the data
alter table jointbl_test set fileformat sequencefile
---- RESULTS
+'Updated table.'
+---- TYPES
+string
====
---- QUERY
# now the proper data should be returned
@@ -333,10 +336,16 @@ alter table t_part add partition (j=NULL, s=NULL)
# be read twice.
alter table t_part drop partition (j=100, s='same_location')
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
alter table t_part drop partition (j=101, s='different_part_dir')
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
insert overwrite table t_part partition(j=NULL, s=NULL)
@@ -392,6 +401,9 @@ int,int,string
---- QUERY
alter table t_part drop partition (j=NULL, s=NULL)
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
select i, j, s from t_part
@@ -413,6 +425,9 @@ int,int,string
---- QUERY
alter table t_part drop partition (j=NULL, s=trim(' 2013 '))
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
select i, j, s from t_part
@@ -429,6 +444,9 @@ int,int,string
---- QUERY
alter table t_part drop partition (j=1, s='2013')
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
select i, j, s from t_part
@@ -442,6 +460,9 @@ int,int,string
---- QUERY
alter table t_part drop partition (j=1, s='2012/withslash')
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
select i, j, s from t_part
@@ -480,6 +501,9 @@ int,int,string
# Drop default partition using empty string as key
alter table t_part drop partition (j=2, s='')
---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
# Validate previous partition drop
@@ -528,6 +552,9 @@ set location '$FILESYSTEM_PREFIX/test-warehouse/alltypes_seq_snap/year=2009/mont
alter table alltypes_test partition(month=4, year=2009)
set fileformat sequencefile
---- RESULTS
+'Updated 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
select int_col, count(*) from alltypes_test
@@ -557,6 +584,9 @@ set location '$FILESYSTEM_PREFIX/test-warehouse/alltypes_rc/year=2009/month=5'
alter table alltypes_test partition(month=cast(2+3 as int), year=2009)
set fileformat rcfile
---- RESULTS
+'Updated 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
select int_col, count(*) from alltypes_test
@@ -591,12 +621,18 @@ STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
# Test altering the 'numRows' table property of a table.
alter table alltypes_test set tblproperties ('numRows'='200')
---- RESULTS
+'Updated table.'
+---- TYPES
+string
====
---- QUERY
# Test altering the 'numRows' table property of a partition.
alter table alltypes_test partition(year=2009, month=4)
set tblproperties ('numRows'='30', 'STATS_GENERATED_VIA_STATS_TASK'='true')
---- RESULTS
+'Updated 1 partition(s).'
+---- TYPES
+string
====
---- QUERY
# Show the table stats after altering the table and partition stats.
@@ -770,6 +806,9 @@ set serdeproperties(
'valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluev
aluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalueval
uevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue
valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalueva
luevaluevalue'
)
---- RESULTS
+'Updated table.'
+---- TYPES
+string
====
---- QUERY
# alter with property lengths just within limits
@@ -780,6 +819,9 @@ set tblproperties(
'valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluev
aluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalueval
uevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalue
valuevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevaluevalueva
luevaluevalue'
)
---- RESULTS
+'Updated table.'
+---- TYPES
+string
====
---- QUERY
-- Change table location after a partition has been created in the original table location
@@ -926,6 +968,9 @@ STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING, STRING,
create table i1740_alter_1 (i1 integer);
alter table i1740_alter_1 set tblproperties ('skip.header.line.count'='2');
---- RESULTS
+'Updated table.'
+---- TYPES
+string
====
---- QUERY
# IMPALA-1740: Test setting the skip.header.line.count tblproperty
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
index a5ddb90..817dd4d 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/compute-stats.test
@@ -262,6 +262,7 @@ STRING, STRING, BIGINT, BIGINT, INT, DOUBLE
# Drop the partitions with NULL values and check for stats.
alter table alltypes drop partition (year=NULL, month=NULL)
---- RESULTS
+'Dropped 1 partition(s).'
====
---- QUERY
show column stats alltypes
@@ -287,6 +288,7 @@ STRING, STRING, BIGINT, BIGINT, INT, DOUBLE
---- QUERY
alter table alltypes drop partition (year=2011, month=NULL)
---- RESULTS
+'Dropped 1 partition(s).'
====
---- QUERY
show column stats alltypes
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test b/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
index 9477958..8896c3a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/hdfs-caching.test
@@ -358,7 +358,7 @@ location 'file://$IMPALA_HOME/testdata/data/cached_local_part'
# Turning on caching for this new partition should fail
alter table cached_tbl_local partition(j=0) set cached in 'testPool'
---- CATCH
-Partition (j=0) cannot be cached. Please check if the table or partitions are on a filesystem which supports caching.
+Partition(s) (j=0) cannot be cached. Please check if the table or partitions are on a filesystem which supports caching.
====
---- QUERY
# Turning on caching should also fail for this table
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/testdata/workloads/functional-query/queries/QueryTest/kudu_alter.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/kudu_alter.test b/testdata/workloads/functional-query/queries/QueryTest/kudu_alter.test
index e6814e1..4572a5f 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/kudu_alter.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/kudu_alter.test
@@ -10,6 +10,9 @@ alter table simple set tblproperties (
'kudu.master_addresses' = 'localhost'
)
---- RESULTS
+'Updated table.'
+---- TYPES
+STRING
====
---- QUERY
-- Show that new address is picked up
@@ -22,6 +25,9 @@ STRING,STRING,STRING
---- QUERY
alter table simple set tblproperties ('kudu.master_addresses' = '127.0.0.1')
---- RESULTS
+'Updated table.'
+---- TYPES
+STRING
====
---- QUERY
-- Try to use an invalid master address
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
new file mode 100644
index 0000000..cc7e865
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/partition-ddl-predicates.test
@@ -0,0 +1,156 @@
+====
+---- QUERY
+# First create a partitioned table
+create table p1 (i int) partitioned by(j int, k string);
+# Add some partitions
+alter table p1 add partition (j=1,k="a");
+alter table p1 add partition (j=1,k="b");
+alter table p1 add partition (j=1,k="c");
+alter table p1 add partition (j=2,k="d");
+alter table p1 add partition (j=2,k="e");
+alter table p1 add partition (j=2,k="f");
+alter table p1 add partition (j=2,k=NULL);
+alter table p1 add partition (j=NULL,k="g");
+alter table p1 add partition (j=NULL,k=NULL);
+# Populate the table
+insert into p1 partition (j, k) values (100, 1, "a"), (200, 1, "b"), (300, 1, "c");
+====
+---- QUERY
+show partitions p1
+---- RESULTS
+'NULL','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=__HIVE_DEFAULT_PARTITION__
+'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
+'1','a',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=a
+'1','b',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=b
+'1','c',-1,1,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=1/k=c
+'2','NULL',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=__HIVE_DEFAULT_PARTITION__
+'2','d',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
+'2','e',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
+'2','f',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
+'Total','',-1,3,regex:.+,regex:.+,'','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
+---- QUERY
+show files in p1 partition (j<2, k="a")
+---- RESULTS
+regex:.+,regex:.+,'j=1/k=a'
+---- TYPES
+STRING, STRING, STRING
+====
+---- QUERY
+alter table p1 partition (j<2, k in ("b", "c")) set cached in 'testPool'
+---- RESULTS
+'Cached 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (j<2, j>0, k<>"d") set uncached
+---- RESULTS
+'Uncached 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (j=3 or j=2, k like "%") set uncached
+---- RESULTS
+'Uncached 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (j=2) set fileformat parquet
+---- RESULTS
+'Updated 4 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (k rlike ".*") set serdeproperties ('k'='v')
+---- RESULTS
+'Updated 7 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (k="") set serdeproperties ('k'='v')
+---- RESULTS
+'Updated 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (k=null) set serdeproperties ('k'='v')
+---- RESULTS
+'Updated 2 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 partition (j is not null) set tblproperties ('k'='v')
+---- RESULTS
+'Updated 7 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+compute incremental stats p1 partition(j<2)
+---- RESULTS
+'Updated 3 partition(s) and 1 column(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j<2)
+---- RESULTS
+'Dropped 3 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j<2)
+---- RESULTS
+'Dropped 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop if exists partition (j<2)
+---- RESULTS
+'Dropped 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop if exists partition (j=2, k="bla")
+---- RESULTS
+'Dropped 0 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j=2, k=NULL)
+---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+alter table p1 drop partition (j IS NULL AND k IS NULL);
+---- RESULTS
+'Dropped 1 partition(s).'
+---- TYPES
+STRING
+====
+---- QUERY
+show partitions p1
+---- RESULTS
+'NULL','g',-1,0,regex:.+,regex:.+,regex:.+,regex:.+,regex:.+,regex:.*/test-warehouse/.+/p1/j=__HIVE_DEFAULT_PARTITION__/k=g
+'2','d',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=d
+'2','e',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=e
+'2','f',-1,0,regex:.+,regex:.+,regex:.+,'PARQUET',regex:.+,regex:.*/test-warehouse/.+/p1/j=2/k=f
+'Total','',0,0,regex:.+,regex:.+,'','','',''
+---- TYPES
+STRING, STRING, BIGINT, BIGINT, STRING, STRING, STRING, STRING, STRING, STRING
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/628685ae/tests/metadata/test_ddl.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_ddl.py b/tests/metadata/test_ddl.py
index 8079855..15fe4c7 100644
--- a/tests/metadata/test_ddl.py
+++ b/tests/metadata/test_ddl.py
@@ -391,6 +391,10 @@ class TestDdlStatements(TestDdlBase):
assert properties['p2'] == 'val3'
assert properties[''] == ''
+ @UniqueDatabase.parametrize(sync_ddl=True)
+ def test_partition_ddl_predicates(self, vector, unique_database):
+ self.run_test_case('QueryTest/partition-ddl-predicates', vector,
+ use_db=unique_database, multiple_impalad=self._use_multiple_impalad(vector))
# IMPALA-2002: Tests repeated adding/dropping of .jar and .so in the lib cache.
class TestLibCache(TestDdlBase):
[7/9] incubator-impala git commit: IMPALA-4372: 'Describe formatted'
returns types in upper case
Posted by sa...@apache.org.
IMPALA-4372: 'Describe formatted' returns types in upper case
A recent change caused 'describe formatted' to display the types
in all upper case, but we want 'describe formatted' to match Hive's
'describe' output, which displays the types in lower case.
This patch also fixes several problems with test_describe_formatted,
which was encountering an error but reporting success.
Change-Id: I274b97d4d1247244247fb38a5ca7f4c10bba8d22
Reviewed-on: http://gerrit.cloudera.org:8080/4861
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/e6e2baea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/e6e2baea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/e6e2baea
Branch: refs/heads/master
Commit: e6e2baea33eca0fbad9958c7c2f6087ee1019c46
Parents: 0ea4a66
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
Authored: Wed Oct 26 19:41:19 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 05:38:12 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/catalog/Column.java | 2 +-
.../queries/QueryTest/avro-schema-changes.test | 6 +--
tests/common/impala_test_suite.py | 9 ++--
.../metadata/test_metadata_query_statements.py | 47 +++++++++++++++-----
tests/performance/query_exec_functions.py | 10 ++---
tests/performance/query_executor.py | 1 +
6 files changed, 49 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6e2baea/fe/src/main/java/org/apache/impala/catalog/Column.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Column.java b/fe/src/main/java/org/apache/impala/catalog/Column.java
index ab064e7..91928aa 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Column.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Column.java
@@ -124,7 +124,7 @@ public class Column {
return Lists.transform(columns, new Function<Column, FieldSchema>() {
public FieldSchema apply(Column column) {
Preconditions.checkNotNull(column.getType());
- return new FieldSchema(column.getName(), column.getType().toSql(),
+ return new FieldSchema(column.getName(), column.getType().toSql().toLowerCase(),
column.getComment());
}
});
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6e2baea/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
index 1e3eac9..8233a02 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
@@ -101,8 +101,8 @@ DESCRIBE FORMATTED avro_alter_schema_add_new_column;
---- TYPES
string,string,string
---- RESULTS: VERIFY_IS_SUBSET
-'old_col','STRING','from deserializer'
-'new_col','STRING','from deserializer'
+'old_col','string','from deserializer'
+'new_col','string','from deserializer'
====
---- QUERY
# IMPALA-3776: Create an Avro table, remove a column from the Avro schema and make sure
@@ -134,5 +134,5 @@ DESCRIBE FORMATTED avro_alter_schema_remove_column;
---- TYPES
string,string,string
---- RESULTS: VERIFY_IS_SUBSET
-'col1','STRING','from deserializer'
+'col1','string','from deserializer'
====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6e2baea/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index afa8e2c..e287360 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -467,24 +467,23 @@ class ImpalaTestSuite(BaseTestSuite):
assert len(result.data) <= 1, 'Multiple values returned from scalar'
return result.data[0] if len(result.data) == 1 else None
- def exec_and_compare_hive_and_impala_hs2(self, stmt):
+ def exec_and_compare_hive_and_impala_hs2(self, stmt, compare = lambda x, y: x == y):
"""Compare Hive and Impala results when executing the same statment over HS2"""
# execute_using_jdbc expects a Query object. Convert the query string into a Query
# object
query = Query()
query.query_str = stmt
# Run the statement targeting Hive
- exec_opts = JdbcQueryExecConfig(impalad=HIVE_HS2_HOST_PORT)
+ exec_opts = JdbcQueryExecConfig(impalad=HIVE_HS2_HOST_PORT, transport='SASL')
hive_results = execute_using_jdbc(query, exec_opts).data
# Run the statement targeting Impala
- exec_opts = JdbcQueryExecConfig(impalad=IMPALAD_HS2_HOST_PORT)
+ exec_opts = JdbcQueryExecConfig(impalad=IMPALAD_HS2_HOST_PORT, transport='NOSASL')
impala_results = execute_using_jdbc(query, exec_opts).data
# Compare the results
assert (impala_results is not None) and (hive_results is not None)
- for impala, hive in zip(impala_results, hive_results):
- assert impala == hive
+ assert compare(impala_results, hive_results)
def load_query_test_file(self, workload, file_name, valid_section_names=None,
encoding=None):
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6e2baea/tests/metadata/test_metadata_query_statements.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_metadata_query_statements.py b/tests/metadata/test_metadata_query_statements.py
index 0a51acb..8e25a2a 100644
--- a/tests/metadata/test_metadata_query_statements.py
+++ b/tests/metadata/test_metadata_query_statements.py
@@ -18,6 +18,7 @@
# Impala tests for queries that query metadata and set session settings
import pytest
+import re
from tests.beeswax.impala_beeswax import ImpalaBeeswaxException
from tests.common.impala_test_suite import ImpalaTestSuite
@@ -75,13 +76,36 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
@SkipIfS3.hive
@SkipIfLocal.hive
def test_describe_formatted(self, vector, unique_database):
+ # For describe formmated, we try to match Hive's output as closely as possible.
+ # However, we're inconsistent with our handling of NULLs vs theirs - Impala sometimes
+ # specifies 'NULL' where Hive uses an empty string, and Hive somtimes specifies 'null'
+ # with padding where Impala uses a sequence of blank spaces - and for now
+ # we want to leave it that way to not affect users who rely on this output.
+ def compare_describe_formatted(impala_results, hive_results):
+ for impala, hive in zip(re.split(',|\n', impala_results),
+ re.split(',|\n', hive_results)):
+
+ if impala != hive:
+ # If they don't match, check if it's because of the inconsistent null handling.
+ impala = impala.replace(' ', '').lower()
+ hive = hive.replace(' ', '').lower()
+ if not ((impala == "'null'" and hive == "''") or
+ (impala == "''" and hive == "'null'")):
+ return False
+ return True
+
# Describe a partitioned table.
- self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.alltypes")
+ self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.alltypes",
+ compare=compare_describe_formatted)
self.exec_and_compare_hive_and_impala_hs2(
- "describe formatted functional_text_lzo.alltypes")
+ "describe formatted functional_text_lzo.alltypes",
+ compare=compare_describe_formatted)
+
# Describe an unpartitioned table.
- self.exec_and_compare_hive_and_impala_hs2("describe formatted tpch.lineitem")
- self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.jointbl")
+ self.exec_and_compare_hive_and_impala_hs2("describe formatted tpch.lineitem",
+ compare=compare_describe_formatted)
+ self.exec_and_compare_hive_and_impala_hs2("describe formatted functional.jointbl",
+ compare=compare_describe_formatted)
# Create and describe an unpartitioned and partitioned Avro table created
# by Impala without any column definitions.
@@ -91,20 +115,19 @@ class TestMetadataQueryStatements(ImpalaTestSuite):
self.client.execute((
"create table %s.%s with serdeproperties ('avro.schema.url'='%s') stored as avro"
% (unique_database, "avro_alltypes_nopart", self.AVRO_SCHEMA_LOC)))
- self.exec_and_compare_hive_and_impala_hs2("describe formatted avro_alltypes_nopart")
+ self.exec_and_compare_hive_and_impala_hs2("describe formatted avro_alltypes_nopart",
+ compare=compare_describe_formatted)
self.client.execute((
"create table %s.%s partitioned by (year int, month int) "
"with serdeproperties ('avro.schema.url'='%s') stored as avro"
% (unique_database, "avro_alltypes_part", self.AVRO_SCHEMA_LOC)))
- self.exec_and_compare_hive_and_impala_hs2("describe formatted avro_alltypes_part")
+ self.exec_and_compare_hive_and_impala_hs2("describe formatted avro_alltypes_part",
+ compare=compare_describe_formatted)
- try:
- # Describe a view
- self.exec_and_compare_hive_and_impala_hs2(\
- "describe formatted functional.alltypes_view_sub")
- except AssertionError:
- pytest.xfail("Investigate minor difference in displaying null vs empty values")
+ self.exec_and_compare_hive_and_impala_hs2(\
+ "describe formatted functional.alltypes_view_sub",
+ compare=compare_describe_formatted)
@pytest.mark.execute_serially # due to data src setup/teardown
def test_show_data_sources(self, vector):
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6e2baea/tests/performance/query_exec_functions.py
----------------------------------------------------------------------
diff --git a/tests/performance/query_exec_functions.py b/tests/performance/query_exec_functions.py
index 3abcae2..f7d00ec 100644
--- a/tests/performance/query_exec_functions.py
+++ b/tests/performance/query_exec_functions.py
@@ -245,7 +245,7 @@ def execute_using_jdbc(query, query_config):
cmd = query_config.jdbc_client_cmd + " -q \"%s\"" % query_string
return run_query_capture_results(cmd, query, exit_on_error=False)
-def parse_jdbc_query_results(stdout, stderr):
+def parse_jdbc_query_results(stdout, stderr, query):
"""
Parse query execution results for the Impala JDBC client
@@ -260,10 +260,10 @@ def parse_jdbc_query_results(stdout, stderr):
time_taken = float(('%s.%s') % (match.group(1), match.group(2)))
break
result_data = re.findall(r'\[START\]----\n(.*?)\n----\[END\]', stdout, re.DOTALL)[0]
- return create_exec_result(time_taken, result_data)
+ return create_exec_result(time_taken, result_data, query)
-def create_exec_result(time_taken, result_data):
- exec_result = HiveQueryResult()
+def create_exec_result(time_taken, result_data, query):
+ exec_result = HiveQueryResult(query)
if result_data:
LOG.debug('Data:\n%s\n' % result_data)
exec_result.data = result_data
@@ -296,7 +296,7 @@ def run_query_capture_results(cmd, query, exit_on_error):
exec_result.query_error = msg
return exec_result
# The command completed
- exec_result = parse_jdbc_query_results(stdout, stderr)
+ exec_result = parse_jdbc_query_results(stdout, stderr, query)
exec_result.query = query
exec_result.start_time = start_time
return exec_result
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/e6e2baea/tests/performance/query_executor.py
----------------------------------------------------------------------
diff --git a/tests/performance/query_executor.py b/tests/performance/query_executor.py
index 56065b8..5d9fc61 100644
--- a/tests/performance/query_executor.py
+++ b/tests/performance/query_executor.py
@@ -91,6 +91,7 @@ class JdbcQueryExecConfig(ImpalaQueryExecConfig):
Constructed on the fly, since the impalad it points to can change.
"""
+ assert self.transport is not None
return JdbcQueryExecConfig.JDBC_CLIENT_PATH + ' -i "%s" -t %s' % (self._impalad,
self.transport)
[5/9] incubator-impala git commit: IMPALA-4465: Don't hold process
wide lock while serializing Runtime Profile in GetRuntimeProfileStr()
Posted by sa...@apache.org.
IMPALA-4465: Don't hold process wide lock while serializing Runtime Profile in GetRuntimeProfileStr()
This patch changes the code so that the query_exec_state_map_lock_
is not held while serializing the RuntimeProfile, since that is a
pretty expensive operation and happens at least once per query. This
change makes a lot of client calls have less lock contention including
Webserver calls and query registration/unregistration.
Change-Id: I3ad8a1d6644259f177dfb3b29b3ba1ad6a76210a
Reviewed-on: http://gerrit.cloudera.org:8080/5035
Reviewed-by: Sailesh Mukil <sa...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f4a5d863
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f4a5d863
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f4a5d863
Branch: refs/heads/master
Commit: f4a5d863c3123503d8b8c89bf1415f4bbdd6a810
Parents: 628685a
Author: Sailesh <sa...@cloudera.com>
Authored: Thu Nov 10 11:33:31 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 03:33:14 2016 +0000
----------------------------------------------------------------------
be/src/service/impala-server.cc | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f4a5d863/be/src/service/impala-server.cc
----------------------------------------------------------------------
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index 730dfbf..c210e28 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -565,13 +565,13 @@ Status ImpalaServer::GetRuntimeProfileStr(const TUniqueId& query_id,
DCHECK(output != NULL);
// Search for the query id in the active query map
{
- lock_guard<mutex> l(query_exec_state_map_lock_);
- QueryExecStateMap::const_iterator exec_state = query_exec_state_map_.find(query_id);
- if (exec_state != query_exec_state_map_.end()) {
+ shared_ptr<QueryExecState> exec_state = GetQueryExecState(query_id, false);
+ if (exec_state.get() != nullptr) {
+ lock_guard<mutex> l(*exec_state->lock());
if (base64_encoded) {
- exec_state->second->profile().SerializeToArchiveString(output);
+ exec_state->profile().SerializeToArchiveString(output);
} else {
- exec_state->second->profile().PrettyPrint(output);
+ exec_state->profile().PrettyPrint(output);
}
return Status::OK();
}
[9/9] incubator-impala git commit: IMPALA-4434: In Python,
''.split('\n') is [''], which has length 1
Posted by sa...@apache.org.
IMPALA-4434: In Python, ''.split('\n') is [''], which has length 1
This test simply may have never been run in GMT or UTC - it appears to
have an easy-to-make off-by-one error.
Change-Id: Iac4943085b0693deb380499cd0e141eb672bead8
Reviewed-on: http://gerrit.cloudera.org:8080/5061
Reviewed-by: Jim Apple <jb...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/b3cbc960
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/b3cbc960
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/b3cbc960
Branch: refs/heads/master
Commit: b3cbc960a7e4beebf6cb9076a02b3513bb0b2954
Parents: 91b5264
Author: Jim Apple <jb...@cloudera.com>
Authored: Sat Nov 12 18:56:40 2016 -0800
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Tue Nov 15 15:29:26 2016 +0000
----------------------------------------------------------------------
tests/custom_cluster/test_hive_parquet_timestamp_conversion.py | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/b3cbc960/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py b/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
index 6573500..731c766 100644
--- a/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
+++ b/tests/custom_cluster/test_hive_parquet_timestamp_conversion.py
@@ -93,8 +93,10 @@ class TestHiveParquetTimestampConversion(CustomClusterTestSuite):
WHERE h.timestamp_col != FROM_UTC_TIMESTAMP(i.timestamp_col, '%s')
""" % tz_name)\
.get_data()
- expected_row_count = 0 if tz_name in ("UTC", "GMT") else 10000
- assert len(data.split('\n')) == expected_row_count
+ if tz_name in ("UTC", "GMT"):
+ assert len(data) == 0
+ else:
+ assert len(data.split('\n')) == 10000
# A value should either stay null or stay not null.
data = self.execute_query_expect_success(self.client, """
SELECT h.id, h.day, h.timestamp_col, i.timestamp_col