You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2016/10/13 06:58:05 UTC
[1/4] incubator-impala git commit: IMPALA-4047: Remove occurrences of
'CDH'/'cdh' from repo
Repository: incubator-impala
Updated Branches:
refs/heads/master 0449b5bea -> 75a857c0c
IMPALA-4047: Remove occurrences of 'CDH'/'cdh' from repo
This change removes some of the occurrences of the strings 'CDH'/'cdh'
from the Impala repository. References to Cloudera-internal Jiras have
been replaced with upstream Jira issues on issues.cloudera.org.
For several categories of occurrences (e.g. pom.xml files,
DOWNLOAD_CDH_COMPONENTS) I also created a list of follow-up Jiras to
remove the occurrences left after this change.
Change-Id: Icb37e2ef0cd9fa0e581d359c5dd3db7812b7b2c8
Reviewed-on: http://gerrit.cloudera.org:8080/4187
Reviewed-by: Jim Apple <jb...@cloudera.com>
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/ef4c9958
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/ef4c9958
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/ef4c9958
Branch: refs/heads/master
Commit: ef4c9958d018ae0b86cd4318e3288af6a2181c7c
Parents: 0449b5b
Author: Lars Volker <lv...@cloudera.com>
Authored: Wed Aug 31 15:12:12 2016 +0200
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Oct 13 00:40:41 2016 +0000
----------------------------------------------------------------------
.gitignore | 1 -
be/src/exec/hbase-table-scanner.cc | 2 +-
be/src/exprs/expr-test.cc | 2 +-
be/src/scheduling/simple-scheduler.h | 3 ++-
be/src/thirdparty/squeasel/squeasel.h | 4 ++--
be/src/util/redactor.detail.h | 3 +--
be/src/util/redactor.h | 3 +--
bin/build_thirdparty.sh | 1 -
bin/load-data.py | 2 +-
common/thrift/generate_metrics.py | 2 +-
fe/pom.xml | 2 +-
fe/src/main/java/org/apache/impala/analysis/SetStmt.java | 2 +-
.../main/java/org/apache/impala/catalog/HBaseTable.java | 5 ++---
.../org/apache/impala/catalog/HdfsStorageDescriptor.java | 2 +-
.../org/apache/impala/catalog/PartitionStatsUtil.java | 8 ++++----
.../java/org/apache/impala/testutil/ImpalaJdbcClient.java | 3 +--
fe/src/test/resources/postgresql-hive-site.xml.template | 4 ++--
infra/deploy/deploy.py | 4 ++--
testdata/cluster/.gitignore | 1 -
testdata/cluster/admin | 7 +++----
testdata/datasets/functional/schema_constraints.csv | 2 +-
.../functional-planner/queries/PlannerTest/hdfs.test | 2 +-
.../functional-planner/queries/PlannerTest/lineage.test | 10 +++++-----
.../queries/QueryTest/hbase-scan-node.test | 4 ++--
tests/comparison/cluster.py | 2 +-
tests/comparison/discrepancy_searcher.py | 8 ++++----
tests/comparison/leopard/impala_docker_env.py | 5 +++--
tests/comparison/query.py | 4 ++--
tests/custom_cluster/test_insert_behaviour.py | 2 +-
tests/metadata/test_compute_stats.py | 3 ++-
tests/metadata/test_hdfs_encryption.py | 2 +-
tests/query_test/test_decimal_queries.py | 5 +++--
tests/shell/test_shell_commandline.py | 2 +-
tests/stress/concurrent_select.py | 4 ++--
34 files changed, 56 insertions(+), 60 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 4a7f11a..849ee61 100644
--- a/.gitignore
+++ b/.gitignore
@@ -40,7 +40,6 @@ tests/build
tests/results
tests/test-hive-udfs/target/
-cdh-*-hdfs-data/
avro_schemas/
logs/
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/be/src/exec/hbase-table-scanner.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/hbase-table-scanner.cc b/be/src/exec/hbase-table-scanner.cc
index fe8e358..79e45d0 100644
--- a/be/src/exec/hbase-table-scanner.cc
+++ b/be/src/exec/hbase-table-scanner.cc
@@ -167,7 +167,7 @@ Status HBaseTableScanner::Init() {
RETURN_IF_ERROR(
JniUtil::GetGlobalClassRef(env, "org/apache/hadoop/hbase/Cell", &cell_cl_));
} else {
- // Assume a non-CDH5 HBase version because the Cell class wasn't found.
+ // Assume a HBase version prior to 0.95.2 because the Cell class wasn't found.
LOG(INFO) << "Detected HBase version < 0.95.2";
RETURN_IF_ERROR(
JniUtil::GetGlobalClassRef(env, "org/apache/hadoop/hbase/KeyValue",
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/be/src/exprs/expr-test.cc
----------------------------------------------------------------------
diff --git a/be/src/exprs/expr-test.cc b/be/src/exprs/expr-test.cc
index 8064de0..88565e1 100644
--- a/be/src/exprs/expr-test.cc
+++ b/be/src/exprs/expr-test.cc
@@ -3525,7 +3525,7 @@ TEST_F(ExprTest, UnaryOperators) {
// TODO: I think a lot of these casts are not necessary and we should fix this
TEST_F(ExprTest, TimestampFunctions) {
- // Regression test for CDH-19918
+ // Regression test for IMPALA-4209
TestStringValue("cast(from_utc_timestamp(cast(1301180400 as timestamp),"
"'Europe/Moscow') as string)", "2011-03-27 03:00:00");
TestStringValue("cast(from_utc_timestamp(cast(1301180399 as timestamp),"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/be/src/scheduling/simple-scheduler.h
----------------------------------------------------------------------
diff --git a/be/src/scheduling/simple-scheduler.h b/be/src/scheduling/simple-scheduler.h
index b7cc83e..6a9d6db 100644
--- a/be/src/scheduling/simple-scheduler.h
+++ b/be/src/scheduling/simple-scheduler.h
@@ -56,7 +56,8 @@ class SchedulerWrapper;
/// TODO: Notice when there are duplicate statestore registrations (IMPALA-23)
/// TODO: Track assignments (assignment_ctx in ComputeScanRangeAssignment) per query
/// instead of per plan node?
-/// TODO: Remove disable_cached_reads query option in CDH6
+/// TODO: Remove disable_cached_reads query option in the next compatibility-breaking
+/// release (IMPALA-2963)
/// TODO: Replace the usage of shared_ptr with atomic_shared_ptr once compilers support
/// it. Alternatively consider using Kudu's rw locks.
/// TODO: Inject global dependencies into the class (for example ExecEnv::GetInstance(),
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/be/src/thirdparty/squeasel/squeasel.h
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/squeasel/squeasel.h b/be/src/thirdparty/squeasel/squeasel.h
index a17f9ef..68a4034 100644
--- a/be/src/thirdparty/squeasel/squeasel.h
+++ b/be/src/thirdparty/squeasel/squeasel.h
@@ -91,7 +91,7 @@ struct sq_request_info {
// This structure needs to be passed to sq_start(), to let squeasel know
// which callbacks to invoke. For detailed description, see
-// https://github.mtv.cloudera.com/CDH/squeasel/blob/master/UserManual.md
+// https://github.com/cloudera/squeasel/blob/master/UserManual.md
struct sq_callbacks {
// Called when squeasel has received new HTTP request.
// If callback returns non-zero,
@@ -182,7 +182,7 @@ struct sq_callbacks {
// };
// struct sq_context *ctx = sq_start(&my_func, NULL, options);
//
-// Refer to https://github.mtv.cloudera.com/CDH/squeasel/blob/master/UserManual.md
+// Refer to https://github.com/cloudera/squeasel/blob/master/UserManual.md
// for the list of valid option and their possible values.
//
// Return:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/be/src/util/redactor.detail.h
----------------------------------------------------------------------
diff --git a/be/src/util/redactor.detail.h b/be/src/util/redactor.detail.h
index 30b3f71..f0030ad 100644
--- a/be/src/util/redactor.detail.h
+++ b/be/src/util/redactor.detail.h
@@ -19,8 +19,7 @@
/// such as log files or a web ui. For example, this library could be used to log credit
/// card numbers as XXXX-...-XXXX instead of actual numbers.
//
-/// The java original is http://github.mtv.cloudera.com/CDH/logredactor.
-/// TODO: Replace the above link with the public version once it is available.
+/// The java original is https://github.com/cloudera/logredactor .
#ifndef IMPALA_UTIL_REDACTOR_DETAIL_H_
#define IMPALA_UTIL_REDACTOR_DETAIL_H_
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/be/src/util/redactor.h
----------------------------------------------------------------------
diff --git a/be/src/util/redactor.h b/be/src/util/redactor.h
index a3a0c1d..ce8ed79 100644
--- a/be/src/util/redactor.h
+++ b/be/src/util/redactor.h
@@ -19,8 +19,7 @@
/// such as log files or a web ui. For example, this library could be used to log credit
/// card numbers as XXXX-...-XXXX instead of actual numbers.
//
-/// The java original is http://github.mtv.cloudera.com/CDH/logredactor.
-/// TODO: Replace the above link with the public version once it is available.
+/// The java original is https://github.com/cloudera/logredactor .
#ifndef IMPALA_UTIL_REDACTOR_H
#define IMPALA_UTIL_REDACTOR_H
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/bin/build_thirdparty.sh
----------------------------------------------------------------------
diff --git a/bin/build_thirdparty.sh b/bin/build_thirdparty.sh
index 2c965a8..4543225 100755
--- a/bin/build_thirdparty.sh
+++ b/bin/build_thirdparty.sh
@@ -43,7 +43,6 @@ BUILD_SASL=0
BUILD_LDAP=0
BUILD_SNAPPY=0
BUILD_PPROF=0
-BUILD_CDH4EXTRAS=0
BUILD_LZ4=0
for ARG in $*
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/bin/load-data.py
----------------------------------------------------------------------
diff --git a/bin/load-data.py b/bin/load-data.py
index e1bfbbb..df41786 100755
--- a/bin/load-data.py
+++ b/bin/load-data.py
@@ -85,7 +85,7 @@ AVRO_SCHEMA_DIR = "avro_schemas"
GENERATE_SCHEMA_CMD = "generate-schema-statements.py --exploration_strategy=%s "\
"--workload=%s --scale_factor=%s --verbose"
-# Load data using Hive's beeline because the Hive shell has regressed (CDH-17222).
+# Load data using Hive's beeline because the Hive shell has regressed (HIVE-5515).
# The Hive shell is stateful, meaning that certain series of actions lead to problems.
# Examples of problems due to the statefullness of the Hive shell:
# - Creating an HBase table changes the replication factor to 1 for subsequent LOADs.
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/common/thrift/generate_metrics.py
----------------------------------------------------------------------
diff --git a/common/thrift/generate_metrics.py b/common/thrift/generate_metrics.py
index dae92bf..e878a29 100755
--- a/common/thrift/generate_metrics.py
+++ b/common/thrift/generate_metrics.py
@@ -47,7 +47,7 @@ parser.add_option("--output_mdl_path", dest="output_mdl_path",
help="The path of the output mdl file. Default: %default")
# TODO: get default version value from bin/save-version.sh
parser.add_option("--output_mdl_version", dest="output_mdl_version",
- metavar="IMPALA_VERSION", default="2.5.0-cdh5",
+ metavar="IMPALA_VERSION", default="2.8.0-SNAPSHOT",
help="The Impala version that is written in the output mdl.")
options, args = parser.parse_args()
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/pom.xml
----------------------------------------------------------------------
diff --git a/fe/pom.xml b/fe/pom.xml
index 08f6623..c4e7c44 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -207,7 +207,7 @@ under the License.
<version>${env.IMPALA_THRIFT_JAVA_VERSION}</version>
<!-- libthrift depends httpcore 4.1.3 which does not work with KMS. To workaround
this problem the dependency is excluded here and we explicitly add a newer
- httpcore dependency version. See CDH-22024. TODO: Find a better fix. -->
+ httpcore dependency version. See IMPALA-4210. TODO: Find a better fix. -->
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/SetStmt.java b/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
index 87c6727..d307ca7 100644
--- a/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/SetStmt.java
@@ -27,7 +27,7 @@ public class SetStmt extends StatementBase {
private final String key_;
private final String value_;
- // This key is deprecated in CDH5.2; COMPRESSION_CODEC_KEY replaces this
+ // This key is deprecated in Impala 2.0; COMPRESSION_CODEC_KEY replaces this
private static final String DEPRECATED_PARQUET_CODEC_KEY = "PARQUET_COMPRESSION_CODEC";
private static final String COMPRESSION_CODEC_KEY = "COMPRESSION_CODEC";
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java b/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
index 3930610..2d1fc78 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HBaseTable.java
@@ -126,7 +126,7 @@ public class HBaseTable extends Table {
private final static Configuration hbaseConf_ = HBaseConfiguration.create();
// Cached column families. Used primarily for speeding up row stats estimation
- // (see CDH-19292).
+ // (see IMPALA-4211).
private HColumnDescriptor[] columnFamilies_ = null;
protected HBaseTable(TableId id, org.apache.hadoop.hive.metastore.api.Table msTbl,
@@ -715,8 +715,7 @@ public class HBaseTable extends Table {
/**
* This is copied from org.apache.hadoop.hbase.client.HTable. The only difference is
* that it does not use cache when calling getRegionLocation.
- * TODO: Remove this function and use HTable.getRegionsInRange when the non-cache
- * version has been ported to CDH (DISTRO-477).
+ * TODO: Remove this function and use HTable.getRegionsInRange (IMPALA-4082).
* Get the corresponding regions for an arbitrary range of keys.
* <p>
*
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
index 93121c0..ace00c2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsStorageDescriptor.java
@@ -60,7 +60,7 @@ public class HdfsStorageDescriptor {
"org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe", // (rc)
"parquet.hive.serde.ParquetHiveSerDe", // (parquet - legacy)
// TODO: Verify the following Parquet SerDe works with Impala and add
- // support for the new input/output format classes. See CDH-17085.
+ // support for the new input/output format classes. See IMPALA-4214.
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe"); // (parquet)
private final static Logger LOG = LoggerFactory.getLogger(HdfsStorageDescriptor.class);
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/src/main/java/org/apache/impala/catalog/PartitionStatsUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/PartitionStatsUtil.java b/fe/src/main/java/org/apache/impala/catalog/PartitionStatsUtil.java
index 222f252..8285e9b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/PartitionStatsUtil.java
+++ b/fe/src/main/java/org/apache/impala/catalog/PartitionStatsUtil.java
@@ -90,10 +90,10 @@ public class PartitionStatsUtil {
return;
}
- // The HMS has a 4k (as of CDH5.2) limit on the length of any parameter string. The
- // serialised version of the partition stats is often larger than this. Therefore, we
- // naively 'chunk' the byte string into 4k pieces, and store the number of pieces in a
- // separate parameter field.
+ // The HMS has a 4k (as of Hive 0.13, Impala 2.0) limit on the length of any parameter
+ // string. The serialised version of the partition stats is often larger than this.
+ // Therefore, we naively 'chunk' the byte string into 4k pieces, and store the number
+ // of pieces in a separate parameter field.
//
// The object itself is first serialised by Thrift, and then base-64 encoded to be a
// valid string. This inflates its length somewhat; we may want to consider a
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/src/test/java/org/apache/impala/testutil/ImpalaJdbcClient.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/testutil/ImpalaJdbcClient.java b/fe/src/test/java/org/apache/impala/testutil/ImpalaJdbcClient.java
index 7176342..1d34ef0 100644
--- a/fe/src/test/java/org/apache/impala/testutil/ImpalaJdbcClient.java
+++ b/fe/src/test/java/org/apache/impala/testutil/ImpalaJdbcClient.java
@@ -284,8 +284,7 @@ public class ImpalaJdbcClient {
float seconds = (endTime - startTime) / 1000F;
LOG.info("Returned " + rowCount + " row(s) in " + seconds + "s");
- // TODO: To work around a JDBC driver issue (CDH-10035), make sure the Statement
- // is closed after every query.
+ // Make sure the Statement is closed after every query.
client.getStatement().close();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/fe/src/test/resources/postgresql-hive-site.xml.template
----------------------------------------------------------------------
diff --git a/fe/src/test/resources/postgresql-hive-site.xml.template b/fe/src/test/resources/postgresql-hive-site.xml.template
index d25d7fa..c89abca 100644
--- a/fe/src/test/resources/postgresql-hive-site.xml.template
+++ b/fe/src/test/resources/postgresql-hive-site.xml.template
@@ -167,7 +167,7 @@
</property>
-->
-<!-- Start CDH-17946 -->
+<!-- Start IMPALA-4125 -->
<property>
<name>hbase.security.authentication</name>
<value>kerberos</value>
@@ -187,7 +187,7 @@
<name>hbase.zookeeper.quorum</name>
<value>localhost</value>
</property>
-<!-- END CDH-17946 -->
+<!-- END IMPALA-4125 -->
<property>
<name>hbase.coprocessor.region.classes</name>
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/infra/deploy/deploy.py
----------------------------------------------------------------------
diff --git a/infra/deploy/deploy.py b/infra/deploy/deploy.py
index c4f40d9..56e3c46 100644
--- a/infra/deploy/deploy.py
+++ b/infra/deploy/deploy.py
@@ -324,7 +324,7 @@ def create_new_service(api, cluster, new_name, deps, scratch_dirs, master_host):
print "Created new role %s" % (new_role_name,)
def transform_path(rcg_name, rcg_config_dict, rcg_config_name):
- # XXX: Do a better job with paths where the role type is embedded.
+ # TODO: Do a better job with paths where the role type is embedded.
#
# e.g. /var/log/impalad/lineage --> /var/log/impalad2/lineage
val = rcg_config_dict.get(rcg_config_name, None)
@@ -335,7 +335,7 @@ def transform_path(rcg_name, rcg_config_dict, rcg_config_name):
return {rcg_config_name : new_val}
def transform_port(rcg_name, rcg_config_dict, rcg_config_name):
- # XXX: Actually resolve all port conflicts.
+ # TODO: Actually resolve all port conflicts.
val = rcg_config_dict.get(rcg_config_name, None)
if not val:
raise Exception("Could not get %s config for rcg %s" %
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/testdata/cluster/.gitignore
----------------------------------------------------------------------
diff --git a/testdata/cluster/.gitignore b/testdata/cluster/.gitignore
index c362112..a8aeb3e 100644
--- a/testdata/cluster/.gitignore
+++ b/testdata/cluster/.gitignore
@@ -1,2 +1 @@
-/cdh*
/kudu
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/testdata/cluster/admin
----------------------------------------------------------------------
diff --git a/testdata/cluster/admin b/testdata/cluster/admin
index 98969ca..bbb4755 100755
--- a/testdata/cluster/admin
+++ b/testdata/cluster/admin
@@ -20,13 +20,12 @@
# This will create/control/destroy a local hdfs+yarn cluster.
#
-# The original idea was to run each node on a different loopback address but
-# https://jira.cloudera.com/browse/CDH-16602 makes that impossible for now. So all roles
-# run on 127.0.0.1, just like the standard mini cluster included with hadoop. The
-# difference is with this cluster, each role runs in its own process and has its own
+# All roles run on 127.0.0.1, just like the standard mini cluster included with hadoop.
+# The difference is with this cluster, each role runs in its own process and has its own
# configs. For each node, the layout of the configs, logs, start/stop scripts, etc, is
# kept as close as possible to a real cluster. For example, the first node will live in
# the dir "cdh-<version>/node-1" and its logs would be at "cdh-<version>/node-1/var/log".
+# TODO: Run each node on its own IP address, e.g. 127.0.0.1, 127.0.0.2, and so on.
set -euo pipefail
trap 'echo Error in $0 at line $LINENO: $(awk "NR == $LINENO" $0)' ERR
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/testdata/datasets/functional/schema_constraints.csv
----------------------------------------------------------------------
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index a22a432..4c9da6d 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -123,7 +123,7 @@ table_name:widerow, constraint:exclude, table_format:hbase/none/none
table_name:nullformat_custom, constraint:exclude, table_format:hbase/none/none
table_name:unsupported_types, constraint:exclude, table_format:hbase/none/none
-# On CDH4, decimal can only be tested on formats Impala can write to (text and parquet)
+# Decimal can only be tested on formats Impala can write to (text and parquet).
# TODO: add Avro once Hive or Impala can write Avro decimals
table_name:decimal_tbl, constraint:restrict_to, table_format:text/none/none
table_name:decimal_tiny, constraint:restrict_to, table_format:text/none/none
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
index 880017b..785d111 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/hdfs.test
@@ -623,7 +623,7 @@ NODE 0:
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=8/100108.txt 0:76263
HDFS SPLIT hdfs://localhost:20500/test-warehouse/alltypesagg/year=2010/month=1/day=__HIVE_DEFAULT_PARTITION__/000000_0 0:72759
====
-# Predicates on a partition key with no values (see CDH-20089)
+# Predicates on a partition key with no values (see IMPALA-4128).
select * from functional.emptytable where f2 = 10
---- PLAN
00:SCAN HDFS [functional.emptytable]
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test b/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
index 9ba2ab7..20c117e 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/lineage.test
@@ -3435,7 +3435,7 @@ from
]
}
====
-# Create view statement (CDH-26149)
+# Create view statement (IMPALA-4219)
create view test_view_lineage as select id from functional.alltypestiny
---- LINEAGE
{
@@ -3468,7 +3468,7 @@ create view test_view_lineage as select id from functional.alltypestiny
]
}
====
-# Create view with nested inline views (CDH-26149)
+# Create view with nested inline views (IMPALA-4219)
create view test_view_lineage (a, b) as select c1, x2
from (
select c1, min(c2) x2
@@ -3547,7 +3547,7 @@ limit 0
]
}
====
-# Create view with complex select statement (CDH-26149)
+# Create view with complex select statement (IMPALA-4219)
create view test_view_lineage (a1, a2, a3, a4, a5, a6, a7) as
select x.smallint_col, x.id, x.tinyint_col, c.id, x.int_col, x.float_col, c.string_col
from functional.alltypessmall c
@@ -3734,7 +3734,7 @@ create view test_view_lineage (a1, a2, a3, a4, a5, a6, a7) as
]
}
====
-# Create view with select statement containing an analytic function (CDH-26149)
+# Create view with select statement containing an analytic function (IMPALA-4219)
create view test_view_lineage as
select * from (
select sum(a.tinyint_col) over (partition by a.smallint_col order by a.id),
@@ -3878,7 +3878,7 @@ create view test_view_lineage as
]
}
====
-# Alter view statement (CDH-26149)
+# Alter view statement (IMPALA-4219)
alter view functional.alltypes_view as select id from functional.alltypestiny
---- LINEAGE
{
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/testdata/workloads/functional-query/queries/QueryTest/hbase-scan-node.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hbase-scan-node.test b/testdata/workloads/functional-query/queries/QueryTest/hbase-scan-node.test
index f23fa56..55415eb 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/hbase-scan-node.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/hbase-scan-node.test
@@ -636,7 +636,7 @@ select * from alltypessmallbinary
INT, BIGINT, BOOLEAN, STRING, DOUBLE, FLOAT, INT, INT, SMALLINT, STRING, TIMESTAMP, TINYINT, INT
====
---- QUERY
-# Scan an HBase table with multiple column families (CDH-18969)
+# Scan an HBase table with multiple column families (IMPALA-4220)
select * from hbasecolumnfamilies
---- RESULTS
0,true,0,0,0,0,'01/01/09',0,0,'0',2009-01-01 00:00:00
@@ -649,4 +649,4 @@ select * from hbasecolumnfamilies
7,false,1,1,1,10,'04/01/09',10.1,1.100000023841858,'1',2009-04-01 00:01:00
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, STRING, DOUBLE, FLOAT, STRING, TIMESTAMP
-====
\ No newline at end of file
+====
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/comparison/cluster.py
----------------------------------------------------------------------
diff --git a/tests/comparison/cluster.py b/tests/comparison/cluster.py
index bfb8fa4..ab1e4f3 100644
--- a/tests/comparison/cluster.py
+++ b/tests/comparison/cluster.py
@@ -630,7 +630,7 @@ class Impalad(object):
try:
self._request_web_page("/cancel_query", params={"query_id": id})
except requests.exceptions.HTTPError as e:
- # XXX: Handle losing the race
+ # TODO: Handle losing the race
raise e
def shell(self, cmd, timeout_secs=DEFAULT_TIMEOUT):
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/comparison/discrepancy_searcher.py
----------------------------------------------------------------------
diff --git a/tests/comparison/discrepancy_searcher.py b/tests/comparison/discrepancy_searcher.py
index a1e60e2..57e284a 100755
--- a/tests/comparison/discrepancy_searcher.py
+++ b/tests/comparison/discrepancy_searcher.py
@@ -247,7 +247,7 @@ class QueryResultComparator(object):
class QueryExecutor(object):
'''Concurrently executes queries'''
- # XXX: Set to false while IMPALA-3336 is a problem. Disabling random query options
+ # TODO: Set to false while IMPALA-3336 is a problem. Disabling random query options
# seems to reduce IMPALA-3336 occurances.
ENABLE_RANDOM_QUERY_OPTIONS = False
@@ -379,9 +379,9 @@ class QueryExecutor(object):
cursor.conn.kill()
LOG.debug('Kill connection')
try:
- # XXX: Sometimes this takes a very long time causing the program to appear to
- # hang. Maybe this should be done in another thread so a timeout can be
- # applied?
+ # TODO: Sometimes this takes a very long time causing the program to appear to
+ # hang. Maybe this should be done in another thread so a timeout can be
+ # applied?
cursor.close()
except Exception as e:
LOG.info('Error closing cursor: %s', e)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/comparison/leopard/impala_docker_env.py
----------------------------------------------------------------------
diff --git a/tests/comparison/leopard/impala_docker_env.py b/tests/comparison/leopard/impala_docker_env.py
index 3f1d292..715ad65 100755
--- a/tests/comparison/leopard/impala_docker_env.py
+++ b/tests/comparison/leopard/impala_docker_env.py
@@ -32,7 +32,7 @@ import os
IMPALA_HOME = '/home/dev/Impala'
CORE_PATH = '/tmp/core_files'
DEFAULT_BRANCH_NAME = 'origin/cdh5-trunk'
-DEFAULT_DOCKER_IMAGE_NAME = 'impala-desktop.ca.cloudera.com:5000/ubuntu-14.04:cdh5-trunk'
+DEFAULT_DOCKER_IMAGE_NAME = 'cloudera/impala-dev'
DOCKER_USER_NAME = 'dev'
NUM_START_ATTEMPTS = 50
NUM_FABRIC_ATTEMPTS = 50
@@ -152,10 +152,11 @@ class ImpalaDockerEnv(object):
elif SHOULD_BUILD_IMPALA:
build_command = (
'docker-boot && cd {IMPALA_HOME} '
- '&& git fetch --all && git checkout origin/cdh5-trunk '
+ '&& git fetch --all && git checkout DEFAULT_BRANCH_NAME '
'&& source {IMPALA_HOME}/bin/impala-config.sh '
'&& {IMPALA_HOME}/buildall.sh -notests').format(
IMPALA_HOME = IMPALA_HOME,
+ DEFAULT_BRANCH_NAME = DEFAULT_BRANCH_NAME,
CORE_PATH = CORE_PATH)
if build_command:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/comparison/query.py
----------------------------------------------------------------------
diff --git a/tests/comparison/query.py b/tests/comparison/query.py
index b928bf3..f6ec65c 100644
--- a/tests/comparison/query.py
+++ b/tests/comparison/query.py
@@ -336,8 +336,8 @@ class Subquery(ValExpr):
result set.
'''
- # XXX: So far it seems fine to use this class for both scalar/non scalar cases but
- # this could lead to unexpected behavior or be a silent cause of problems...
+ # TODO: So far it seems fine to use this class for both scalar/non scalar cases but
+ # this could lead to unexpected behavior or be a silent cause of problems...
def __init__(self, query):
self.query = query
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/custom_cluster/test_insert_behaviour.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_insert_behaviour.py b/tests/custom_cluster/test_insert_behaviour.py
index 9cec88f..6df8ed7 100644
--- a/tests/custom_cluster/test_insert_behaviour.py
+++ b/tests/custom_cluster/test_insert_behaviour.py
@@ -120,7 +120,7 @@ class TestInsertBehaviourCustomCluster(CustomClusterTestSuite):
self.execute_query_expect_success(client, "INSERT INTO %s"
" PARTITION(p1=1, p2=3, p3=4) VALUES(1)" % TEST_TBL)
# Would be 777 if inheritance was enabled
- if not IS_ISILON: # CDH-27688
+ if not IS_ISILON: # IMPALA-4221
self._check_partition_perms("p1=1/p2=3/", default_perms)
self._check_partition_perms("p1=1/p2=3/p3=4/", default_perms)
finally:
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/metadata/test_compute_stats.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_compute_stats.py b/tests/metadata/test_compute_stats.py
index 6f6d31a..9ecdcdf 100644
--- a/tests/metadata/test_compute_stats.py
+++ b/tests/metadata/test_compute_stats.py
@@ -42,7 +42,8 @@ class TestComputeStats(ImpalaTestSuite):
@SkipIfLocal.hdfs_blocks
def test_compute_stats(self, vector, unique_database):
self.run_test_case('QueryTest/compute-stats', vector, unique_database)
- # Test compute stats on decimal columns separately so we can vary between CDH4/5
+ # Test compute stats on decimal columns separately so we can vary between platforms
+ # with and without write support for decimals (Hive < 0.11 and >= 0.11).
self.run_test_case('QueryTest/compute-stats-decimal', vector, unique_database)
def test_compute_stats_incremental(self, vector, unique_database):
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/metadata/test_hdfs_encryption.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_hdfs_encryption.py b/tests/metadata/test_hdfs_encryption.py
index 196b2ea..a257de0 100644
--- a/tests/metadata/test_hdfs_encryption.py
+++ b/tests/metadata/test_hdfs_encryption.py
@@ -146,7 +146,7 @@ class TestHdfsEncryption(ImpalaTestSuite):
def test_drop_partition_encrypt(self):
"""Verifies if alter <tbl> drop partition purge works in case
where the Trash dir and partition dir are in different encryption
- zones. Check CDH-31350 for details"""
+ zones. Check IMPALA-2310 for details"""
self.client.execute("create table {0}.t1(i int) partitioned\
by (j int)".format(TEST_DB))
# Add three partitions (j=1), (j=2), (j=3) to table t1
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/query_test/test_decimal_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_decimal_queries.py b/tests/query_test/test_decimal_queries.py
index 0404ea1..f95d940 100644
--- a/tests/query_test/test_decimal_queries.py
+++ b/tests/query_test/test_decimal_queries.py
@@ -35,8 +35,9 @@ class TestDecimalQueries(ImpalaTestSuite):
cls.TestMatrix.add_dimension(
TestDimension('batch_size', *TestDecimalQueries.BATCH_SIZES))
- # On CDH4, hive does not support decimal so we can't run these tests against
- # the other file formats. Enable them on C5.
+ # Hive < 0.11 does not support decimal so we can't run these tests against the other
+ # file formats.
+ # TODO: Enable them on Hive >= 0.11.
cls.TestMatrix.add_constraint(lambda v:\
(v.get_value('table_format').file_format == 'text' and
v.get_value('table_format').compression_codec == 'none') or
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/shell/test_shell_commandline.py
----------------------------------------------------------------------
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 31cf570..16a68ae 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -200,7 +200,7 @@ class TestImpalaShell(ImpalaTestSuite):
assert 'Problem parsing file' in result.stderr
def test_no_warnings_in_log_with_quiet_mode(self):
- """Regression test for CDH-21036."""
+ """Regression test for IMPALA-4222."""
args = ('-q "set abort_on_error=false;'
' select count(*) from functional_seq_snap.bad_seq_snap" --quiet')
result = run_impala_shell_cmd(args)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/ef4c9958/tests/stress/concurrent_select.py
----------------------------------------------------------------------
diff --git a/tests/stress/concurrent_select.py b/tests/stress/concurrent_select.py
index 1d1d142..b76d713 100755
--- a/tests/stress/concurrent_select.py
+++ b/tests/stress/concurrent_select.py
@@ -335,8 +335,8 @@ class StressRunner(object):
If a query completes without error, the result will be verified. An error
will be raised upon a result mismatch.
"""
- # XXX: The state from a previous run should be cleared out. This isn't really a
- # problem now because the one caller (main()) never calls a second time.
+ # TODO: The state from a previous run should be cleared out. This isn't really a
+ # problem now because the one caller (main()) never calls a second time.
if self.startup_queries_per_sec <= 0:
raise Exception("Startup queries per second must be positive")
[4/4] incubator-impala git commit: IMPALA-4259: build Impala without
any test cluster setup.
Posted by ta...@apache.org.
IMPALA-4259: build Impala without any test cluster setup.
The main outcome of this change is to avoid making unnecessary
modification to the Impala or other source trees when we don't need the
test cluster.
To achieve that, this refactors the script to make the flow easier
to understand and makes it more consistent which build steps are
executed in which modes.
Change-Id: I429da7bc6681b16c07fe58bb3efac6d1a8579137
Reviewed-on: http://gerrit.cloudera.org:8080/4685
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/75a857c0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/75a857c0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/75a857c0
Branch: refs/heads/master
Commit: 75a857c0ceb6a58691ff45f0df8d88586edc8acd
Parents: 0b3efb1
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Sun Oct 9 23:06:40 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Oct 13 05:45:47 2016 +0000
----------------------------------------------------------------------
bin/clean.sh | 7 +-
buildall.sh | 319 +++++++++++++++++++-----------------
ext-data-source/CMakeLists.txt | 4 +-
fe/CMakeLists.txt | 4 +-
4 files changed, 176 insertions(+), 158 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/75a857c0/bin/clean.sh
----------------------------------------------------------------------
diff --git a/bin/clean.sh b/bin/clean.sh
index ec5a658..7d5b315 100755
--- a/bin/clean.sh
+++ b/bin/clean.sh
@@ -26,12 +26,7 @@ trap 'echo Error in $0 at line $LINENO: $(cd "'$PWD'" && awk "NR == $LINENO" $0)
# If the project was never build, no Makefile will exist and thus make clean will fail.
# Combine the make command with the bash noop to always return true.
-make clean || :
-
-# Stop the minikdc if needed.
-if "${CLUSTER_DIR}/admin" is_kerberized; then
- "${IMPALA_HOME}/testdata/bin/minikdc.sh" stop
-fi
+"${MAKE_CMD:-make}" clean || :
# clean the external data source project
pushd ${IMPALA_HOME}/ext-data-source
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/75a857c0/buildall.sh
----------------------------------------------------------------------
diff --git a/buildall.sh b/buildall.sh
index a71c521..ebd3eb3 100755
--- a/buildall.sh
+++ b/buildall.sh
@@ -29,7 +29,7 @@ ROOT=`cd "$ROOT" >/dev/null; pwd`
# kerberized environment variables already or not.
NEEDS_RE_SOURCE_NOTE=1
: ${MINIKDC_REALM=}
-if [ ! -z "${MINIKDC_REALM}" ]; then
+if [[ ! -z "${MINIKDC_REALM}" ]]; then
NEEDS_RE_SOURCE_NOTE=0
fi
@@ -46,6 +46,7 @@ TESTS_ACTION=1
FORMAT_CLUSTER=0
FORMAT_METASTORE=0
FORMAT_SENTRY_POLICY_DB=0
+START_IMPALA_CLUSTER=0
IMPALA_KERBERIZE=0
SNAPSHOT_FILE=
METASTORE_SNAPSHOT_FILE=
@@ -62,7 +63,6 @@ LZO_CMAKE_ARGS=
: ${CMAKE_BUILD_TYPE:=Debug}
# parse command line options
-# TODO: We have to change this to use getopts, or something more sensible.
while [ -n "$*" ]
do
case "$1" in
@@ -113,9 +113,9 @@ do
;;
-snapshot_file)
SNAPSHOT_FILE="${2-}"
- if [ ! -f "$SNAPSHOT_FILE" ]; then
+ if [[ ! -f "$SNAPSHOT_FILE" ]]; then
echo "-snapshot_file does not exist: $SNAPSHOT_FILE"
- exit 1;
+ exit 1
fi
TESTDATA_ACTION=1
# Get the full path.
@@ -124,15 +124,18 @@ do
;;
-metastore_snapshot_file)
METASTORE_SNAPSHOT_FILE="${2-}"
- if [ ! -f "$METASTORE_SNAPSHOT_FILE" ]; then
+ if [[ ! -f "$METASTORE_SNAPSHOT_FILE" ]]; then
echo "-metastore_snapshot_file does not exist: $METASTORE_SNAPSHOT_FILE"
- exit 1;
+ exit 1
fi
TESTDATA_ACTION=1
# Get the full path.
METASTORE_SNAPSHOT_FILE="$(readlink -f "$METASTORE_SNAPSHOT_FILE")"
shift;
;;
+ -start_impala_cluster)
+ START_IMPALA_CLUSTER=1
+ ;;
-k|-kerberize|-kerberos|-kerb)
# Export to the environment for all child process tools
export IMPALA_KERBERIZE=1
@@ -166,7 +169,9 @@ do
echo "[-asan] : Address sanitizer build [Default: False]"
echo "[-skiptests] : Skips execution of all tests"
echo "[-notests] : Skips building and execution of all tests"
- echo "[-testpairwise] : Sun tests in 'pairwise' mode (increases"\
+ echo "[-start_impala_cluster] : Start Impala minicluster after build"\
+ " [Default: False]"
+ echo "[-testpairwise] : Run tests in 'pairwise' mode (increases"\
"test execution time)"
echo "[-testexhaustive] : Run tests in 'exhaustive' mode (significantly increases"\
"test execution time)"
@@ -228,13 +233,21 @@ if [[ ${BUILD_ASAN} -eq 1 ]]; then
fi
CMAKE_BUILD_TYPE=ADDRESS_SANITIZER
fi
+MAKE_IMPALA_ARGS+=" -build_type=${CMAKE_BUILD_TYPE}"
# If we aren't kerberized then we certainly don't need to talk about
# re-sourcing impala-config.
-if [ ${IMPALA_KERBERIZE} -eq 0 ]; then
+if [[ ${IMPALA_KERBERIZE} -eq 0 ]]; then
NEEDS_RE_SOURCE_NOTE=0
fi
+if [[ ${IMPALA_KERBERIZE} -eq 1 &&
+ (${TESTDATA_ACTION} -eq 1 || ${TESTS_ACTION} -eq 1) ]]; then
+ echo "Running tests or loading test data is not supported for kerberized clusters."
+ echo "Please remove the -testdata flag and/or add the -skiptests flag."
+ exit 1
+fi
+
# Loading data on a filesystem other than fs.defaultFS is not supported.
if [[ -z "$METASTORE_SNAPSHOT_FILE" && "${TARGET_FILESYSTEM}" != "hdfs" &&
"$TESTDATA_ACTION" -eq 1 ]]; then
@@ -243,160 +256,130 @@ if [[ -z "$METASTORE_SNAPSHOT_FILE" && "${TARGET_FILESYSTEM}" != "hdfs" &&
exit 1
fi
-# option to clean everything first
-if [ "$CLEAN_ACTION" -eq 1 ]; then
- "$IMPALA_HOME/bin/clean.sh"
+NEED_MINICLUSTER=0
+if [[ $TESTS_ACTION -eq 1 || $TESTDATA_ACTION -eq 1 || $FORMAT_CLUSTER -eq 1 ||
+ $FORMAT_METASTORE -eq 1 || $FORMAT_SENTRY_POLICY_DB -eq 1 || -n "$SNAPSHOT_FILE" ||
+ -n "$METASTORE_SNAPSHOT_FILE" ]]; then
+ NEED_MINICLUSTER=1
fi
-# Populate necessary thirdparty components unless it's set to be skipped.
-if [ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]; then
- echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping download of Python dependencies."
- echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap."
-else
- echo "Downloading Python dependencies"
- # Download all the Python dependencies we need before doing anything
- # of substance. Does not re-download anything that is already present.
- if ! "$IMPALA_HOME/infra/python/deps/download_requirements"; then
- echo "Warning: Unable to download Python requirements."
- echo "Warning: bootstrap_virtualenv or other Python-based tooling may fail."
+bootstrap_dependencies() {
+ # Populate necessary thirdparty components unless it's set to be skipped.
+ if [[ "${SKIP_TOOLCHAIN_BOOTSTRAP}" = true ]]; then
+ echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping download of Python dependencies."
+ echo "SKIP_TOOLCHAIN_BOOTSTRAP is true, skipping toolchain bootstrap."
else
- echo "Finished downloading Python dependencies"
+ echo "Downloading Python dependencies"
+ # Download all the Python dependencies we need before doing anything
+ # of substance. Does not re-download anything that is already present.
+ if ! "$IMPALA_HOME/infra/python/deps/download_requirements"; then
+ echo "Warning: Unable to download Python requirements."
+ echo "Warning: bootstrap_virtualenv or other Python-based tooling may fail."
+ else
+ echo "Finished downloading Python dependencies"
+ fi
+
+ echo "Downloading and extracting toolchain dependencies."
+ "$IMPALA_HOME/bin/bootstrap_toolchain.py"
+ echo "Toolchain bootstrap complete."
fi
+}
- echo "Downloading and extracting toolchain dependencies."
- "$IMPALA_HOME/bin/bootstrap_toolchain.py"
- echo "Toolchain bootstrap complete."
-fi
-
-MAKE_IMPALA_ARGS="${MAKE_IMPALA_ARGS} -build_type=${CMAKE_BUILD_TYPE}"
-
-if [ "$BUILD_FE_ONLY" -eq 1 ]; then
+# Build the Impala frontend and its dependencies.
+build_fe() {
"$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS} -cmake_only
"${MAKE_CMD}" fe
- exit 0
-fi
-
-if [ -e "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.so ]
-then
- cp "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.* "$HADOOP_HOME/lib/native"
-else
- echo "No hadoop-lzo found"
-fi
-
-# Stop any running Impala services.
-"${IMPALA_HOME}/bin/start-impala-cluster.py" --kill --force
-
-if [[ "$CLEAN_ACTION" -eq 1 || "$FORMAT_METASTORE" -eq 1 || "$FORMAT_CLUSTER" -eq 1 ||
- "$FORMAT_SENTRY_POLICY_DB" -eq 1 || -n "$METASTORE_SNAPSHOT_FILE" ]]
-then
- # Kill any processes that may be accessing postgres metastore. To be safe, this is done
- # before we make any changes to the config files.
- set +e
- "${IMPALA_HOME}/testdata/bin/kill-all.sh"
- set -e
-fi
-
-CREATE_TEST_CONFIG_ARGS=""
-if [[ "$FORMAT_SENTRY_POLICY_DB" -eq 1 ]]; then
- CREATE_TEST_CONFIG_ARGS+=" -create_sentry_policy_db"
-fi
-
-if [[ "$FORMAT_METASTORE" -eq 1 && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
- CREATE_TEST_CONFIG_ARGS+=" -create_metastore"
-fi
-
-# Generate the Hadoop configs needed by Impala
-"${IMPALA_HOME}/bin/create-test-configuration.sh" ${CREATE_TEST_CONFIG_ARGS}
-
-# If a metastore snapshot exists, load it.
-if [ "$METASTORE_SNAPSHOT_FILE" ]; then
- echo "Loading metastore snapshot"
- "${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh" "$METASTORE_SNAPSHOT_FILE"
-fi
+}
+
+# Build all components.
+build_all_components() {
+ # Build common and backend. This also sets up the CMake files.
+ echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}"
+ "$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
+
+ if [[ -e "$IMPALA_LZO" ]]
+ then
+ pushd "$IMPALA_LZO"
+ LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
+ rm -f CMakeCache.txt
+ cmake ${LZO_CMAKE_ARGS}
+ "${MAKE_CMD}"
+ popd
+ fi
-# build common and backend
-echo "Calling make_impala.sh ${MAKE_IMPALA_ARGS}"
-"$IMPALA_HOME/bin/make_impala.sh" ${MAKE_IMPALA_ARGS}
-
-if [ -e "$IMPALA_LZO" ]
-then
- pushd "$IMPALA_LZO"
- LZO_CMAKE_ARGS+=" -DCMAKE_TOOLCHAIN_FILE=./cmake_modules/toolchain.cmake"
- rm -f CMakeCache.txt
- cmake ${LZO_CMAKE_ARGS}
- "${MAKE_CMD}"
+ # Build the Java components (fe and external data source API).
+ pushd "$IMPALA_HOME"
+ "${MAKE_CMD}" ext-data-source fe
popd
-fi
-# build the external data source API
-pushd "${IMPALA_HOME}/ext-data-source"
-"${IMPALA_HOME}/bin/mvn-quiet.sh" install -DskipTests
-popd
+ # Build the shell tarball
+ echo "Creating shell tarball"
+ "${IMPALA_HOME}/shell/make_shell_tarball.sh"
+
+ # Generate list of files for Cscope to index
+ "$IMPALA_HOME/bin/gen-cscope.sh"
+}
+
+# Do any configuration of the test cluster required by the script arguments.
+# Kills any cluster processes that will need to be restarted to pick up new
+# configurations or the new build.
+reconfigure_test_cluster() {
+ # Stop any running Impala services.
+ "${IMPALA_HOME}/bin/start-impala-cluster.py" --kill --force
+
+ if [[ "$FORMAT_METASTORE" -eq 1 || "$FORMAT_CLUSTER" -eq 1 ||
+ "$FORMAT_SENTRY_POLICY_DB" -eq 1 || -n "$METASTORE_SNAPSHOT_FILE" ]]
+ then
+ # Kill any processes that may be accessing postgres metastore. To be safe, this is
+ # done before we make any changes to the config files.
+ "${IMPALA_HOME}/testdata/bin/kill-all.sh" || true
+ fi
-# build frontend and copy dependencies
-pushd "${IMPALA_FE_DIR}"
-"${IMPALA_HOME}/bin/mvn-quiet.sh" package -DskipTests
-popd
+ # Stop the minikdc if needed.
+ if "${CLUSTER_DIR}/admin" is_kerberized; then
+ "${IMPALA_HOME}/testdata/bin/minikdc.sh" stop
+ fi
-# Build the shell tarball
-echo "Creating shell tarball"
-"${IMPALA_HOME}/shell/make_shell_tarball.sh"
+ local CREATE_TEST_CONFIG_ARGS=""
+ if [[ "$FORMAT_SENTRY_POLICY_DB" -eq 1 ]]; then
+ CREATE_TEST_CONFIG_ARGS+=" -create_sentry_policy_db"
+ fi
-if [ "$FORMAT_CLUSTER" -eq 1 ]; then
- "$IMPALA_HOME/testdata/bin/run-all.sh" -format
-elif [ "$TESTDATA_ACTION" -eq 1 ] || [ "$TESTS_ACTION" -eq 1 ]; then
- "$IMPALA_HOME/testdata/bin/run-all.sh"
-fi
+ if [[ "$FORMAT_METASTORE" -eq 1 && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
+ CREATE_TEST_CONFIG_ARGS+=" -create_metastore"
+ fi
-#
-# KERBEROS TODO
-# There is still work to be done for kerberos.
-# - The hive metastore needs to be kerberized
-# - If the user principal is "impala/localhost", MR jobs complain that user
-# "impala" is not user ${USER}. But if the principal is ${USER}/localhost,
-# the impala daemons change it to impala/localhost in
-# KerberosAuthProvider::RunKinit() - and there may be other difficulties
-# down the road with getting all the permissions correct.
-# - Futher Beeline -> HiveServer2 -> HBase|MapReduce combo issues
-# - Getting farther down the testing path, it's likely more issues will turn up
-# - Further extensive testing
-#
-if [ ${IMPALA_KERBERIZE} -eq 1 ]; then
- if [ ${TESTDATA_ACTION} -eq 1 -o ${TESTS_ACTION} -eq 1 ]; then
- echo "At this time we only support cluster creation and impala daemon"
- echo "bringup in kerberized mode. Data won't be loaded, and tests"
- echo "won't be run. The impala daemons will be started."
- TESTDATA_ACTION=0
- TESTS_ACTION=0
- "${IMPALA_HOME}/bin/start-impala-cluster.py"
+ # Generate the Hadoop configs needed by Impala
+ "${IMPALA_HOME}/bin/create-test-configuration.sh" ${CREATE_TEST_CONFIG_ARGS}
+
+ # Copy Hadoop-lzo dependencies if available (required to generate Lzo data).
+ if stat "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.* > /dev/null ; then
+ cp "$HADOOP_LZO"/build/native/Linux-*-*/lib/libgplcompression.* "$HADOOP_HOME/lib/native"
+ else
+ echo "No hadoop-lzo found"
fi
-fi
-# END KERBEROS TODO
+}
-#
-# Don't try to run tests without data!
-#
-TESTWH_ITEMS=`(hadoop fs -ls ${FILESYSTEM_PREFIX}/test-warehouse 2> /dev/null || true) | \
- (grep test-warehouse || true) | wc -l`
-if [ ${TESTS_ACTION} -eq 1 -a \
- ${TESTDATA_ACTION} -eq 0 -a \
- ${TESTWH_ITEMS} -lt 5 ]; then
- set +x
- echo "You just asked buildall to run tests, but did not supply any data."
- echo "Running tests without data doesn't work. Exiting now."
- exit 1
-fi
+# Starts the test cluster processes except for Impala.
+start_test_cluster_dependencies() {
+ local RUN_ALL_ARGS=""
+ if [[ "$FORMAT_CLUSTER" -eq 1 ]]; then
+ RUN_ALL_ARGS+=" -format"
+ fi
+ "$IMPALA_HOME/testdata/bin/run-all.sh" $RUN_ALL_ARGS
+}
-if [ $TESTDATA_ACTION -eq 1 ]; then
- # Create testdata.
+# Execute any data loading steps once the cluster dependencies are started.
+# This does all data loading, except for the metastore snapshot which must be loaded
+# earlier before the cluster is running.
+load_test_data() {
"$IMPALA_HOME/bin/create_testdata.sh"
- cd "$ROOT"
# We have 4 cases:
# - test-warehouse and metastore snapshots exists.
# - Only the test-warehouse snapshot exists.
# - Only the metastore snapshot exists.
# - Neither of them exist.
- CREATE_LOAD_DATA_ARGS=""
+ local CREATE_LOAD_DATA_ARGS=""
if [[ "$SNAPSHOT_FILE" && "$METASTORE_SNAPSHOT_FILE" ]]; then
CREATE_LOAD_DATA_ARGS="-snapshot_file ${SNAPSHOT_FILE} -skip_metadata_load"
elif [[ "$SNAPSHOT_FILE" && -z "$METASTORE_SNAPSHOT_FILE" ]]; then
@@ -405,20 +388,60 @@ if [ $TESTDATA_ACTION -eq 1 ]; then
CREATE_LOAD_DATA_ARGS="-skip_metadata_load -skip_snapshot_load"
fi
"${IMPALA_HOME}/testdata/bin/create-load-data.sh" ${CREATE_LOAD_DATA_ARGS} <<< Y
-fi
+}
-if [ $TESTS_ACTION -eq 1 ]; then
- if [ $CODE_COVERAGE -eq 0 ]; then
- "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY
- else
- "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY -c
+run_all_tests() {
+ local RUN_ALL_TESTS_ARGS=
+ if [[ $CODE_COVERAGE -eq 1 ]]; then
+ RUN_ALL_TESTS_ARGS+=" -c"
fi
+ "${IMPALA_HOME}/bin/run-all-tests.sh" -e $EXPLORATION_STRATEGY $RUN_ALL_TESTS_ARGS
+}
+
+# Clean everything first if requested.
+if [[ "$CLEAN_ACTION" -eq 1 ]]; then
+ "$IMPALA_HOME/bin/clean.sh"
+fi
+
+bootstrap_dependencies
+
+if [[ "$BUILD_FE_ONLY" -eq 1 ]]; then
+ build_fe
+ exit 0
+fi
+
+build_all_components
+
+if [[ $NEED_MINICLUSTER -eq 1 ]]; then
+ reconfigure_test_cluster
+fi
+
+# If a metastore snapshot exists, load it while the cluster process are down and not
+# accessing the metastore.
+if [[ -n "$METASTORE_SNAPSHOT_FILE" ]]; then
+ echo "Loading metastore snapshot"
+ "${IMPALA_HOME}/testdata/bin/load-metastore-snapshot.sh" "$METASTORE_SNAPSHOT_FILE"
fi
-# Generate list of files for Cscope to index
-"$IMPALA_HOME/bin/gen-cscope.sh"
+if [[ $NEED_MINICLUSTER -eq 1 ]]; then
+ start_test_cluster_dependencies
+fi
+
+if [[ $TESTDATA_ACTION -eq 1 ]]; then
+ load_test_data
+fi
+
+if [[ $TESTS_ACTION -eq 1 ]]; then
+ run_all_tests
+fi
+
+# Bring up Impala if requested. Tests and data load start their own miniclusters, so we
+# should bring up a clean cluster *after* those steps are completed.
+if [[ $START_IMPALA_CLUSTER -eq 1 ]]; then
+ "${IMPALA_HOME}/bin/start-impala-cluster.py"
+fi
-if [ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]; then
+if [[ ${NEEDS_RE_SOURCE_NOTE} -eq 1 ]]; then
echo
echo "You have just successfully created a kerberized cluster."
echo "Congratulations! Communication with this cluster requires"
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/75a857c0/ext-data-source/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/ext-data-source/CMakeLists.txt b/ext-data-source/CMakeLists.txt
index 6022b2b..d2c1077 100644
--- a/ext-data-source/CMakeLists.txt
+++ b/ext-data-source/CMakeLists.txt
@@ -16,5 +16,5 @@
# under the License.
add_custom_target(ext-data-source DEPENDS thrift-deps
- COMMAND mvn install -DskipTests
-)
\ No newline at end of file
+ COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
+)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/75a857c0/fe/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/fe/CMakeLists.txt b/fe/CMakeLists.txt
index f17c470..c162435 100644
--- a/fe/CMakeLists.txt
+++ b/fe/CMakeLists.txt
@@ -16,5 +16,5 @@
# under the License.
add_custom_target(fe DEPENDS thrift-deps function-registry ext-data-source
- COMMAND mvn package -DskipTests
-)
\ No newline at end of file
+ COMMAND $ENV{IMPALA_HOME}/bin/mvn-quiet.sh install -DskipTests
+)
[3/4] incubator-impala git commit: IMPALA-4289: Mark agg slots of
NDV() functions as non-nullable.
Posted by ta...@apache.org.
IMPALA-4289: Mark agg slots of NDV() functions as non-nullable.
This change might give a minor speedup to COMPUTE STATS
and COMPUTE INCREMENTAL STATS. In any case, marking the slots
non-nullable seems strictly better than leaving them nullable.
Testing: I ran our local testdata/compute-table-stats.sh and
it succeeded.
Change-Id: I1c05b8dfb797b2a42ee1a7bf14ad56bb83d2b1c5
Reviewed-on: http://gerrit.cloudera.org:8080/4707
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0b3efb19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0b3efb19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0b3efb19
Branch: refs/heads/master
Commit: 0b3efb19cc0d971a47d0b7d7928a2d311c13e65a
Parents: f2780b5
Author: Alex Behm <al...@cloudera.com>
Authored: Wed Oct 12 16:57:49 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Oct 13 04:31:42 2016 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/analysis/AggregateInfoBase.java | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0b3efb19/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java b/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java
index bced878..51c56ae 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AggregateInfoBase.java
@@ -147,10 +147,13 @@ public abstract class AggregateInfoBase {
slotDesc.setSourceExpr(aggExpr);
}
- // count(*) is non-nullable.
- if (aggExpr.getFnName().getFunction().equals("count")) {
+ // COUNT(), NDV() and NDV_NO_FINALIZE() are non-nullable. The latter two are used
+ // by compute stats and compute incremental stats, respectively.
+ if (aggExpr.getFnName().getFunction().equals("count")
+ || aggExpr.getFnName().getFunction().equals("ndv")
+ || aggExpr.getFnName().getFunction().equals("ndv_no_finalize")) {
// TODO: Consider making nullability a property of types or of builtin agg fns.
- // row_number, rank, and dense_rank are non-nullable as well.
+ // row_number(), rank(), and dense_rank() are non-nullable as well.
slotDesc.setIsNullable(false);
}
if (!isOutputTuple) {
[2/4] incubator-impala git commit: IMPALA-4287: EE tests fail to run
when KUDU_IS_SUPPORTED=false
Posted by ta...@apache.org.
IMPALA-4287: EE tests fail to run when KUDU_IS_SUPPORTED=false
When Kudu tests were added to the functional-query test
dimensions, we needed to make sure the dimension was skipped
when KUDU_IS_SUPPORTED=false.
Change-Id: Ib97572b94c54bb36ad1d7184aa45744e8a42ffa4
Reviewed-on: http://gerrit.cloudera.org:8080/4697
Reviewed-by: Alex Behm <al...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/f2780b56
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/f2780b56
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/f2780b56
Branch: refs/heads/master
Commit: f2780b567b33c9f57ffeefbf82a41d67cd0bb14d
Parents: ef4c995
Author: Matthew Jacobs <mj...@cloudera.com>
Authored: Wed Oct 12 13:35:21 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Thu Oct 13 01:28:10 2016 +0000
----------------------------------------------------------------------
tests/common/test_dimensions.py | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/f2780b56/tests/common/test_dimensions.py
----------------------------------------------------------------------
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index c7bebb4..95195df 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -184,6 +184,10 @@ def load_table_info_dimension(workload_name, exploration_strategy, file_formats=
vals = dict((key.strip(), value.strip()) for key, value in\
(item.split(':') for item in line.split(',')))
+ # Skip Kudu if Kudu is not supported (IMPALA-4287).
+ if os.environ['KUDU_IS_SUPPORTED'] != 'true' and vals['file_format'] == 'kudu':
+ continue
+
# If only loading specific file formats skip anything that doesn't match
if file_formats is not None and vals['file_format'] not in file_formats:
continue