You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by bi...@apache.org on 2018/10/09 23:18:17 UTC
[1/8] impala git commit: IMPALA-7623: [DOCS] Disallow name changes
for managed Kudu tables
Repository: impala
Updated Branches:
refs/heads/master d48ffc2d4 -> 6568e6e11
IMPALA-7623: [DOCS] Disallow name changes for managed Kudu tables
Removed SET TBLPROPERTIES('kudu.table_name') when creating and altering
managed Kudu tables.
Change-Id: I843bf3a6574c2a1ad34cb93d3060f5f097256554
Reviewed-on: http://gerrit.cloudera.org:8080/11518
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Gabor Kaszab <ga...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/51cc1d2c
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/51cc1d2c
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/51cc1d2c
Branch: refs/heads/master
Commit: 51cc1d2cf78433bcb8aa58f2c0a868ff5ac2154e
Parents: d48ffc2
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Tue Sep 25 16:46:12 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Mon Oct 8 17:19:08 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_alter_table.xml | 17 ++---
docs/topics/impala_show.xml | 5 +-
docs/topics/impala_tables.xml | 128 +++++---------------------------
3 files changed, 29 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/51cc1d2c/docs/topics/impala_alter_table.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_alter_table.xml b/docs/topics/impala_alter_table.xml
index 804f4cb..2d0d1c6 100644
--- a/docs/topics/impala_alter_table.xml
+++ b/docs/topics/impala_alter_table.xml
@@ -1007,15 +1007,14 @@ alter table sales_data add partition (zipcode = cast(9021 * 10 as string));</cod
You cannot use the <codeph>REPLACE COLUMNS</codeph> clause with a Kudu table.
</p>
</li>
- <li>
- <p>
- The <codeph>RENAME TO</codeph> clause for a Kudu table only affects the name stored in the
- metastore database that Impala uses to refer to the table. To change which underlying Kudu
- table is associated with an Impala table name, you must change the <codeph>TBLPROPERTIES</codeph>
- property of the table: <codeph>SET TBLPROPERTIES('kudu.table_name'='<varname>kudu_tbl_name</varname>)</codeph>.
- Doing so causes Kudu to change the name of the underlying Kudu table.
- </p>
- </li>
+ <li> The <codeph>RENAME TO</codeph> clause for a Kudu table only affects
+ the name stored in the metastore database that Impala uses to refer to
+ the table. To change which underlying Kudu table is associated with an
+ Impala table name, you must change the <codeph>TBLPROPERTIES</codeph>
+ property of the table: <codeph>SET
+ TBLPROPERTIES('kudu.table_name'='<varname>kudu_tbl_name</varname>)</codeph>.
+ You can only change underlying Kudu tables for the external
+ tables.</li>
</ul>
</p>
http://git-wip-us.apache.org/repos/asf/impala/blob/51cc1d2c/docs/topics/impala_show.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_show.xml b/docs/topics/impala_show.xml
index a25be83..c7e6e0e 100644
--- a/docs/topics/impala_show.xml
+++ b/docs/topics/impala_show.xml
@@ -817,10 +817,9 @@ show CREATE TABLE numeric_grades_default_letter;
| student STRING NULL ENCODING AUTO_ENCODING COMPRESSION DEFAULT_COMPRESSION, |
| PRIMARY KEY (score) |
| ) |
-| PARTITION BY <b>RANGE (score) (...)</b> |
+| PARTITION BY <b>RANGE (score) (...)</b> |
| STORED AS KUDU |
-| TBLPROPERTIES ('kudu.master_addresses'='vd0342.example.com:7051', |
-| 'kudu.table_name'='impala::USER.numeric_grades_default_letter') |
+| TBLPROPERTIES ('kudu.master_addresses'='vd0342.example.com:7051') |
+------------------------------------------------------------------------------------------------+
show range partitions numeric_grades_default_letter;
http://git-wip-us.apache.org/repos/asf/impala/blob/51cc1d2c/docs/topics/impala_tables.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_tables.xml b/docs/topics/impala_tables.xml
index c731cc2..0da7955 100644
--- a/docs/topics/impala_tables.xml
+++ b/docs/topics/impala_tables.xml
@@ -291,115 +291,25 @@ under the License.
managed internally by Kudu.
</p>
- <p>
- When you create a Kudu table through Impala, it is assigned an internal Kudu table name of the form
- <codeph>impala::<varname>db_name</varname>.<varname>table_name</varname></codeph>. You can see the Kudu-assigned name
- in the output of <codeph>DESCRIBE FORMATTED</codeph>, in the <codeph>kudu.table_name</codeph> field of the table properties.
- The Kudu-assigned name remains the same even if you use <codeph>ALTER TABLE</codeph> to rename the Impala table
- or move it to a different Impala database. If you issue the statement
- <codeph>ALTER TABLE <varname>impala_name</varname> SET TBLPROPERTIES('kudu.table_name' = '<varname>different_kudu_table_name</varname>')</codeph>,
- the effect is different depending on whether the Impala table was created with a regular <codeph>CREATE TABLE</codeph>
- statement (that is, if it is an internal or managed table), or if it was created with a
- <codeph>CREATE EXTERNAL TABLE</codeph> statement (and therefore is an external table). Changing the <codeph>kudu.table_name</codeph>
- property of an internal table physically renames the underlying Kudu table to match the new name.
- Changing the <codeph>kudu.table_name</codeph> property of an external table switches which underlying Kudu table
- the Impala table refers to; the underlying Kudu table must already exist.
- </p>
-
- <p>
- The following example shows what happens with both internal and external Kudu tables as the <codeph>kudu.table_name</codeph>
- property is changed. In practice, external tables are typically used to access underlying Kudu tables that were created
- outside of Impala, that is, through the Kudu API.
- </p>
-
-<codeblock>
--- This is an internal table that we will create and then rename.
-create table old_name (id bigint primary key, s string)
- partition by hash(id) partitions 2 stored as kudu;
-
--- Initially, the name OLD_NAME is the same on the Impala and Kudu sides.
-describe formatted old_name;
-...
-| Location: | hdfs://host.example.com:8020/path/user.db/old_name
-| Table Type: | MANAGED_TABLE | NULL
-| Table Parameters: | NULL | NULL
-| | DO_NOT_UPDATE_STATS | true
-| | kudu.master_addresses | vd0342.example.com
-| | kudu.table_name | impala::user.old_name
-
--- ALTER TABLE RENAME TO changes the Impala name but not the underlying Kudu name.
-alter table old_name rename to new_name;
-
-describe formatted new_name;
-| Location: | hdfs://host.example.com:8020/path/user.db/new_name
-| Table Type: | MANAGED_TABLE | NULL
-| Table Parameters: | NULL | NULL
-| | DO_NOT_UPDATE_STATS | true
-| | kudu.master_addresses | vd0342.example.com
-| | kudu.table_name | impala::user.old_name
-
--- Setting TBLPROPERTIES changes the underlying Kudu name.
-alter table new_name
- set tblproperties('kudu.table_name' = 'impala::user.new_name');
-
-describe formatted new_name;
-| Location: | hdfs://host.example.com:8020/path/user.db/new_name
-| Table Type: | MANAGED_TABLE | NULL
-| Table Parameters: | NULL | NULL
-| | DO_NOT_UPDATE_STATS | true
-| | kudu.master_addresses | vd0342.example.com
-| | kudu.table_name | impala::user.new_name
-
--- Put some data in the table to demonstrate how external tables can map to
--- different underlying Kudu tables.
-insert into new_name values (0, 'zero'), (1, 'one'), (2, 'two');
-
--- This external table points to the same underlying Kudu table, NEW_NAME,
--- as we created above. No need to declare columns or other table aspects.
-create external table kudu_table_alias stored as kudu
- tblproperties('kudu.table_name' = 'impala::user.new_name');
-
--- The external table can fetch data from the NEW_NAME table that already
--- existed and already had data.
-select * from kudu_table_alias limit 100;
-+----+------+
-| id | s |
-+----+------+
-| 1 | one |
-| 0 | zero |
-| 2 | two |
-+----+------+
-
--- We cannot re-point the external table at a different underlying Kudu table
--- unless that other underlying Kudu table already exists.
-alter table kudu_table_alias
- set tblproperties('kudu.table_name' = 'impala::user.yet_another_name');
-ERROR:
-TableLoadingException: Error opening Kudu table 'impala::user.yet_another_name',
- Kudu error: The table does not exist: table_name: "impala::user.yet_another_name"
-
--- Once the underlying Kudu table exists, we can re-point the external table to it.
-create table yet_another_name (id bigint primary key, x int, y int, s string)
- partition by hash(id) partitions 2 stored as kudu;
-
-alter table kudu_table_alias
- set tblproperties('kudu.table_name' = 'impala::user.yet_another_name');
-
--- Now no data is returned because this other table is empty.
-select * from kudu_table_alias limit 100;
-
--- The Impala table automatically recognizes the table schema of the new table,
--- for example the extra X and Y columns not present in the original table.
-describe kudu_table_alias;
-+------+--------+---------+-------------+----------+...
-| name | type | comment | primary_key | nullable |...
-+------+--------+---------+-------------+----------+...
-| id | bigint | | true | false |...
-| x | int | | false | true |...
-| y | int | | false | true |...
-| s | string | | false | true |...
-+------+--------+---------+-------------+----------+...
-</codeblock>
+ <p> When you create a Kudu table through Impala, it is assigned an
+ internal Kudu table name of the form
+ <codeph>impala::<varname>db_name</varname>.<varname>table_name</varname></codeph>.
+ You can see the Kudu-assigned name in the output of <codeph>DESCRIBE
+ FORMATTED</codeph>, in the <codeph>kudu.table_name</codeph> field of
+ the table properties. The Kudu-assigned name remains the same even if
+ you use <codeph>ALTER TABLE</codeph> to rename the Impala table or move
+ it to a different Impala database. You can issue the statement
+ <codeph>ALTER TABLE <varname>impala_name</varname> SET
+ TBLPROPERTIES('kudu.table_name' =
+ '<varname>different_kudu_table_name</varname>')</codeph> for the
+ external tables created with the <codeph>CREATE EXTERNAL TABLE</codeph>
+ statement. Changing the <codeph>kudu.table_name</codeph> property of an
+ external table switches which underlying Kudu table the Impala table
+ refers to. The underlying Kudu table must already exist. </p>
+
+ <p>In practice, external tables are typically used to access underlying
+ Kudu tables that were created outside of Impala, that is, through the
+ Kudu API. </p>
<p>
The <codeph>SHOW TABLE STATS</codeph> output for a Kudu table shows Kudu-specific details about the layout of the table.
[2/8] impala git commit: IMPALA-7678: Revert "IMPALA-7660: Support
ECDH ciphers for debug webserver"
Posted by bi...@apache.org.
IMPALA-7678: Revert "IMPALA-7660: Support ECDH ciphers for debug webserver"
This reverts commit 0e1de31ba56bdac73b8db5c5ff316334c84725d9.
Change-Id: Id4034a4323be741bc7d9fffcf17288aeb3649b31
Reviewed-on: http://gerrit.cloudera.org:8080/11616
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/fec2d64e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/fec2d64e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/fec2d64e
Branch: refs/heads/master
Commit: fec2d64e8f4c5adef7cdb746d8ce946cad75acdd
Parents: 51cc1d2
Author: poojanilangekar <po...@cloudera.com>
Authored: Mon Oct 8 10:38:11 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Mon Oct 8 22:19:54 2018 +0000
----------------------------------------------------------------------
be/src/thirdparty/squeasel/squeasel.c | 36 ++++------------------------
tests/common/impala_cluster.py | 15 ++++--------
tests/common/impala_service.py | 26 +++++++-------------
tests/custom_cluster/test_client_ssl.py | 34 +++++---------------------
4 files changed, 25 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/fec2d64e/be/src/thirdparty/squeasel/squeasel.c
----------------------------------------------------------------------
diff --git a/be/src/thirdparty/squeasel/squeasel.c b/be/src/thirdparty/squeasel/squeasel.c
index 045740d..2149497 100644
--- a/be/src/thirdparty/squeasel/squeasel.c
+++ b/be/src/thirdparty/squeasel/squeasel.c
@@ -4298,37 +4298,11 @@ static int set_ssl_option(struct sq_context *ctx) {
(void) SSL_CTX_use_certificate_chain_file(ctx->ssl_ctx, pem);
}
- if (ctx->config[SSL_CIPHERS] != NULL) {
- if (SSL_CTX_set_cipher_list(ctx->ssl_ctx, ctx->config[SSL_CIPHERS]) == 0) {
- cry(fc(ctx), "SSL_CTX_set_cipher_list: error setting ciphers (%s): %s",
- ctx->config[SSL_CIPHERS], ssl_error());
- return 0;
- }
-#ifndef OPENSSL_NO_ECDH
-#if OPENSSL_VERSION_NUMBER < 0x10002000L
- // OpenSSL 1.0.1 and below only support setting a single ECDH curve at once.
- // We choose prime256v1 because it's the first curve listed in the "modern
- // compatibility" section of the Mozilla Server Side TLS recommendations,
- // accessed Feb. 2017.
- EC_KEY* ecdh = EC_KEY_new_by_curve_name(NID_X9_62_prime256v1);
- if (ecdh == NULL) {
- cry(fc(ctx), "EC_KEY_new_by_curve_name: %s", ssl_error());
- }
-
- int rc = SSL_CTX_set_tmp_ecdh(ctx->ssl_ctx, ecdh);
- if (rc <= 0) {
- cry(fc(ctx), "SSL_CTX_set_tmp_ecdh: %s", ssl_error());
- }
-#elif OPENSSL_VERSION_NUMBER < 0x10100000L
- // OpenSSL 1.0.2 provides the set_ecdh_auto API which internally figures out
- // the best curve to use.
- int rc = SSL_CTX_set_ecdh_auto(ctx->ssl_ctx, 1);
- if (rc <= 0) {
- cry(fc(ctx), "SSL_CTX_set_ecdh_auto: %s", ssl_error());
- }
-#endif
-#endif
-
+ if (ctx->config[SSL_CIPHERS] != NULL &&
+ (SSL_CTX_set_cipher_list(ctx->ssl_ctx, ctx->config[SSL_CIPHERS]) == 0)) {
+ cry(fc(ctx), "SSL_CTX_set_cipher_list: error setting ciphers (%s): %s",
+ ctx->config[SSL_CIPHERS], ssl_error());
+ return 0;
}
return 1;
http://git-wip-us.apache.org/repos/asf/impala/blob/fec2d64e/tests/common/impala_cluster.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 05ff172..f25c8ed 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -222,10 +222,6 @@ class BaseImpalaProcess(Process):
def _get_webserver_port(self, default=None):
return int(self._get_arg_value('webserver_port', default))
- def _get_webserver_ssl(self):
- """Returns true if the webserver is being run with ssl."""
- return self._get_arg_value("webserver_certificate_file", "") != ""
-
def _get_arg_value(self, arg_name, default=None):
"""Gets the argument value for given argument name"""
for arg in self.cmd:
@@ -243,8 +239,7 @@ class ImpaladProcess(BaseImpalaProcess):
self.service = ImpaladService(self.hostname, self._get_webserver_port(default=25000),
self.__get_beeswax_port(default=21000),
self.__get_be_port(default=22000),
- self.__get_hs2_port(default=21050),
- self._get_webserver_ssl())
+ self.__get_hs2_port(default=21050))
def __get_beeswax_port(self, default=None):
return int(self._get_arg_value('beeswax_port', default))
@@ -269,16 +264,16 @@ class ImpaladProcess(BaseImpalaProcess):
class StateStoreProcess(BaseImpalaProcess):
def __init__(self, cmd):
super(StateStoreProcess, self).__init__(cmd, socket.gethostname())
- self.service = StateStoredService(
- self.hostname, self._get_webserver_port(default=25010), self._get_webserver_ssl())
+ self.service =\
+ StateStoredService(self.hostname, self._get_webserver_port(default=25010))
# Represents a catalogd process
class CatalogdProcess(BaseImpalaProcess):
def __init__(self, cmd):
super(CatalogdProcess, self).__init__(cmd, socket.gethostname())
- self.service = CatalogdService(self.hostname, self._get_webserver_port(default=25020),
- self._get_webserver_ssl(), self.__get_port(default=26000))
+ self.service = CatalogdService(self.hostname,
+ self._get_webserver_port(default=25020), self.__get_port(default=26000))
def __get_port(self, default=None):
return int(self._get_arg_value('catalog_service_port', default))
http://git-wip-us.apache.org/repos/asf/impala/blob/fec2d64e/tests/common/impala_service.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index a99e0ce..0ad4496 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -22,7 +22,6 @@
import json
import logging
import re
-import ssl
import urllib
from time import sleep, time
@@ -42,24 +41,17 @@ LOG.setLevel(level=logging.DEBUG)
# Base class for all Impala services
# TODO: Refactor the retry/timeout logic into a common place.
class BaseImpalaService(object):
- def __init__(self, hostname, webserver_port, webserver_ssl=False):
+ def __init__(self, hostname, webserver_port):
self.hostname = hostname
self.webserver_port = webserver_port
- self.webserver_ssl = webserver_ssl
def open_debug_webpage(self, page_name, timeout=10, interval=1):
start_time = time()
while (time() - start_time < timeout):
try:
- protocol = "http"
- context = None
- if self.webserver_ssl:
- protocol = "https"
- context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
- url = "%s://%s:%d/%s" % \
- (protocol, self.hostname, int(self.webserver_port), page_name)
- return urllib.urlopen(url, context=context)
+ return urllib.urlopen("http://%s:%d/%s" %
+ (self.hostname, int(self.webserver_port), page_name))
except Exception:
LOG.info("Debug webpage not yet available.")
sleep(interval)
@@ -174,8 +166,8 @@ class BaseImpalaService(object):
# new connections or accessing the debug webpage.
class ImpaladService(BaseImpalaService):
def __init__(self, hostname, webserver_port=25000, beeswax_port=21000, be_port=22000,
- hs2_port=21050, webserver_ssl=False):
- super(ImpaladService, self).__init__(hostname, webserver_port, webserver_ssl)
+ hs2_port=21050):
+ super(ImpaladService, self).__init__(hostname, webserver_port)
self.beeswax_port = beeswax_port
self.be_port = be_port
self.hs2_port = hs2_port
@@ -315,8 +307,8 @@ class ImpaladService(BaseImpalaService):
# Allows for interacting with the StateStore service to perform operations such as
# accessing the debug webpage.
class StateStoredService(BaseImpalaService):
- def __init__(self, hostname, webserver_port, webserver_ssl):
- super(StateStoredService, self).__init__(hostname, webserver_port, webserver_ssl)
+ def __init__(self, hostname, webserver_port):
+ super(StateStoredService, self).__init__(hostname, webserver_port)
def wait_for_live_subscribers(self, num_subscribers, timeout=15, interval=1):
self.wait_for_metric_value('statestore.live-backends', num_subscribers,
@@ -326,8 +318,8 @@ class StateStoredService(BaseImpalaService):
# Allows for interacting with the Catalog service to perform operations such as
# accessing the debug webpage.
class CatalogdService(BaseImpalaService):
- def __init__(self, hostname, webserver_port, webserver_ssl, service_port):
- super(CatalogdService, self).__init__(hostname, webserver_port, webserver_ssl)
+ def __init__(self, hostname, webserver_port, service_port):
+ super(CatalogdService, self).__init__(hostname, webserver_port)
self.service_port = service_port
def get_catalog_version(self, timeout=10, interval=1):
http://git-wip-us.apache.org/repos/asf/impala/blob/fec2d64e/tests/custom_cluster/test_client_ssl.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_client_ssl.py b/tests/custom_cluster/test_client_ssl.py
index 7bbc2df..b6f7e04 100644
--- a/tests/custom_cluster/test_client_ssl.py
+++ b/tests/custom_cluster/test_client_ssl.py
@@ -19,7 +19,6 @@
import logging
import os
import pytest
-import requests
import signal
import ssl
import socket
@@ -111,27 +110,13 @@ class TestClientSsl(CustomClusterTestSuite):
assert "Query Status: Cancelled" in result.stdout
assert impalad.wait_for_num_in_flight_queries(0)
- WEBSERVER_SSL_ARGS = ("--webserver_certificate_file=%(cert_dir)s/server-cert.pem "
- "--webserver_private_key_file=%(cert_dir)s/server-key.pem "
- % {'cert_dir': CERT_DIR})
-
- @pytest.mark.execute_serially
- @CustomClusterTestSuite.with_args(impalad_args=WEBSERVER_SSL_ARGS,
- statestored_args=WEBSERVER_SSL_ARGS,
- catalogd_args=WEBSERVER_SSL_ARGS)
- def test_webserver_ssl(self):
- "Tests that the debug web pages are reachable when run with ssl."
- self._verify_ssl_webserver()
-
# Test that the shell can connect to a ECDH only cluster.
- TLS_ECDH_ARGS = ("--ssl_client_ca_certificate=%(cert_dir)s/server-cert.pem "
- "--ssl_server_certificate=%(cert_dir)s/server-cert.pem "
- "--ssl_private_key=%(cert_dir)s/server-key.pem "
- "--hostname=localhost " # Required to match hostname in certificate"
- "--ssl_cipher_list=ECDHE-RSA-AES128-GCM-SHA256 "
- "--webserver_certificate_file=%(cert_dir)s/server-cert.pem "
- "--webserver_private_key_file=%(cert_dir)s/server-key.pem "
- % {'cert_dir': CERT_DIR})
+ TLS_ECDH_ARGS = ("--ssl_client_ca_certificate=%s/server-cert.pem "
+ "--ssl_server_certificate=%s/server-cert.pem "
+ "--ssl_private_key=%s/server-key.pem "
+ "--hostname=localhost " # Required to match hostname in certificate"
+ "--ssl_cipher_list=ECDHE-RSA-AES128-GCM-SHA256 "
+ % (CERT_DIR, CERT_DIR, CERT_DIR))
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(impalad_args=TLS_ECDH_ARGS,
@@ -143,7 +128,6 @@ class TestClientSsl(CustomClusterTestSuite):
def test_tls_ecdh(self, vector):
self._verify_negative_cases()
self._validate_positive_cases("%s/server-cert.pem" % self.CERT_DIR)
- self._verify_ssl_webserver()
# Test that the shell can connect to a TLS1.2 only cluster, and for good measure
# restrict the cipher suite to just one choice.
@@ -225,9 +209,3 @@ class TestClientSsl(CustomClusterTestSuite):
result = run_impala_shell_cmd(shell_options)
for msg in [self.SSL_ENABLED, self.CONNECTED, self.FETCHED]:
assert msg in result.stderr
-
- def _verify_ssl_webserver(self):
- for port in ["25000", "25010", "25020"]:
- url = "https://localhost:%s" % port
- response = requests.get(url, verify="%s/server-cert.pem" % self.CERT_DIR)
- assert response.status_code == requests.codes.ok, url
[8/8] impala git commit: IMPALA-7607: [DOCS] Add a reference to
EXEC_TIME_LIMIT_S to TIMEOUT doc
Posted by bi...@apache.org.
IMPALA-7607: [DOCS] Add a reference to EXEC_TIME_LIMIT_S to TIMEOUT doc
Change-Id: Iaf7bfd7535ba0f8f51fd0994ad6e9e3f2637b2c1
Reviewed-on: http://gerrit.cloudera.org:8080/11621
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Bikramjeet Vig <bi...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/6568e6e1
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/6568e6e1
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/6568e6e1
Branch: refs/heads/master
Commit: 6568e6e110e507f5228c1da11e56fd7117ab8a25
Parents: 7b13117
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Mon Oct 8 17:09:11 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Tue Oct 9 21:57:12 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_timeouts.xml | 56 ++++++++++++++++++++++--------------
1 file changed, 34 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/6568e6e1/docs/topics/impala_timeouts.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_timeouts.xml b/docs/topics/impala_timeouts.xml
index 3389f5a..0ae394e 100644
--- a/docs/topics/impala_timeouts.xml
+++ b/docs/topics/impala_timeouts.xml
@@ -205,30 +205,42 @@ Trying to re-register with state-store</codeblock>
<conbody>
- <p>
- Sometimes, an Impala query might run for an unexpectedly long time,
- tying up resources in the cluster. You can cancel the query explicitly,
- independent of the timeout period, by going into the web UI for the
- <cmdname>impalad</cmdname> host (on port 25000 by default), and using
- the link on the <codeph>/queries</codeph> tab to cancel the running
- query.
- </p>
-
- <p>
- Various client applications let you interactively cancel queries
- submitted or monitored through those applications. For example:
- <ul>
- <li>
- Press <systemoutput>^C</systemoutput> in
- <cmdname>impala-shell</cmdname>.
- </li>
- <li>
- Click <b>Cancel</b> from the <b>Watch</b>page in Hue.
- </li>
- </ul>
- </p>
+ <p> Sometimes, an Impala query might run for an unexpectedly long time,
+ tying up resources in the cluster. This section describes the options to
+ terminate the runaway queries.</p>
</conbody>
+ <concept id="cancel_query_with_query_option">
+ <title>Setting a Time Limit on Query Execution</title>
+ <conbody>
+ <p>An Impala administrator can set a default value of the
+ <codeph>EXEC_TIME_LIMIT_S</codeph> query option for a resource pool.
+ If a user accidentally runs a large query that executes for longer
+ than the limit, it will be automatically killed after the time limit
+ expires and frees up resources. </p>
+ <p>Users can override the default value per query or per session if they
+ do not want the default <codeph>EXEC_TIME_LIMIT_S</codeph> value to
+ apply to a specific query or a session. See <xref
+ href="impala_exec_time_limit_s.xml#exec_time_limit_s"/> for the
+ details of the query option.</p>
+ </conbody>
+ </concept>
+ <concept id="cancel_query_in_webUI">
+ <title>Interactively Cancelling a Query </title>
+ <conbody>
+ <p> You can cancel the query explicitly, independent of the timeout
+ period, by going into the web UI for the <cmdname>impalad</cmdname>
+ host (on port 25000 by default), and using the link on the
+ <codeph>/queries</codeph> tab to cancel the running query. </p>
+ <p> Various client applications let you interactively cancel queries
+ submitted or monitored through those applications. For example: <ul>
+ <li> Press <systemoutput>^C</systemoutput> in
+ <cmdname>impala-shell</cmdname>. </li>
+ <li> Click <b>Cancel</b> from the <b>Watch</b>page in Hue. </li>
+ </ul>
+ </p>
+ </conbody>
+ </concept>
</concept>
[4/8] impala git commit: IMPALA-7647: Add HS2/Impyla dimension to
TestQueries
Posted by bi...@apache.org.
IMPALA-7647: Add HS2/Impyla dimension to TestQueries
I used some ideas from Alex Leblang's abandoned patch:
https://gerrit.cloudera.org/#/c/137/ in order to run .test files through
HS2. The advantage of using Impyla is that much of the code will be
reusable for any Python client implementing the standard Python dbapi
and does not require us implementing yet another thrift client.
This gives us better coverage of non-trivial result sets from HS2,
including handling of NULLs, error logs and more interesting result
sets than the basic HS2 tests.
I added HS2 coverage to TestQueries, which has a reasonable variety of
queries and covers the data types in alltypes. I also added
TestDecimalQueries, TestStringQuery and TestCharFormats to get coverage
of DECIMAL, CHAR and VARCHAR that aren't in alltypes. Coverage of
results sets with NULLs was limited so I added a couple of queries.
Places where results differ from Beeswax:
* Impyla is a Python dbapi client so must convert timestamps into python datetime
objects, which only have microsecond precision. Therefore result
timestamps within nanosecond precision are truncated.
* The HS2 interface reports the NULL type as BOOLEAN as a workaround for
IMPALA-914.
* The Beeswax interface reported VARCHAR as STRING, but HS2 reports
VARCHAR.
I dealt with different results by adding additional result sections so
that the expected differences between the clients/protocols were
explicit.
Limitations:
* Not all of the same methods are implemented as for beeswax, so some
tests that have more complicated interactions with the client will not
work with HS2 yet.
* We don't have a way to get the affected row count for inserts.
I also simplified the ImpalaConnection API by removing some unnecessary
methods and moved some generic methods to the base class.
Testing:
* Confirmed that it detected IMPALA-7588 by re-applying the buggy patch.
* Ran exhaustive and CentOS6 tests.
Change-Id: I9908ccc4d3df50365be8043b883cacafca52661e
Reviewed-on: http://gerrit.cloudera.org:8080/11546
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/d05f73f4
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/d05f73f4
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/d05f73f4
Branch: refs/heads/master
Commit: d05f73f41588dc1ba08684df63c2da85055be84a
Parents: fec2d64
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Sat Sep 29 12:13:35 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 9 00:45:10 2018 +0000
----------------------------------------------------------------------
bin/start-impala-cluster.py | 16 +-
.../queries/QueryTest/analytic-fns.test | 37 +++
.../queries/QueryTest/chars-formats.test | 4 +
.../queries/QueryTest/chars.test | 46 ++++
.../queries/QueryTest/data-source-tables.test | 10 +
.../queries/QueryTest/decimal.test | 33 +++
.../queries/QueryTest/inline-view-limit.test | 18 ++
.../queries/QueryTest/inline-view.test | 6 +
.../queries/QueryTest/limit.test | 55 +++++
.../queries/QueryTest/top-n.test | 125 ++++++++++
.../queries/QueryTest/union.test | 8 +
tests/beeswax/impala_beeswax.py | 14 +-
tests/common/impala_connection.py | 247 ++++++++++++++++---
tests/common/impala_service.py | 36 ++-
tests/common/impala_test_suite.py | 102 +++++---
tests/common/test_dimensions.py | 20 ++
tests/common/test_result_verifier.py | 46 ++--
tests/conftest.py | 6 +-
tests/custom_cluster/test_coordinators.py | 10 +-
tests/custom_cluster/test_session_expiration.py | 18 +-
tests/metadata/test_views_compatibility.py | 4 +-
tests/query_test/test_aggregation.py | 6 +-
tests/query_test/test_chars.py | 13 +-
tests/query_test/test_decimal_queries.py | 8 +-
tests/query_test/test_insert.py | 6 +-
tests/query_test/test_queries.py | 9 +-
tests/query_test/test_scanners.py | 6 +-
tests/util/test_file_parser.py | 2 +-
28 files changed, 762 insertions(+), 149 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/bin/start-impala-cluster.py
----------------------------------------------------------------------
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index 208cc35..ebf8fa5 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -395,27 +395,25 @@ def wait_for_catalog(impalad, timeout_in_seconds=CLUSTER_WAIT_TIMEOUT_IN_SECONDS
"""Waits for a catalog copy to be received by the impalad. When its received,
additionally waits for client ports to be opened."""
start_time = time()
- client_beeswax = None
- client_hs2 = None
+ beeswax_port_is_open = False
+ hs2_port_is_open = False
num_dbs = 0
num_tbls = 0
- while (time() - start_time < timeout_in_seconds):
+ while ((time() - start_time < timeout_in_seconds) and
+ not (beeswax_port_is_open and hs2_port_is_open)):
try:
num_dbs, num_tbls = impalad.service.get_metric_values(
["catalog.num-databases", "catalog.num-tables"])
- client_beeswax = impalad.service.create_beeswax_client()
- client_hs2 = impalad.service.create_hs2_client()
- break
+ beeswax_port_is_open = impalad.service.beeswax_port_is_open()
+ hs2_port_is_open = impalad.service.hs2_port_is_open()
except Exception as e:
LOG.exception(("Client services not ready. Waiting for catalog cache: "
"({num_dbs} DBs / {num_tbls} tables). Trying again ...").format(
num_dbs=num_dbs,
num_tbls=num_tbls))
- finally:
- if client_beeswax is not None: client_beeswax.close()
sleep(0.5)
- if client_beeswax is None or client_hs2 is None:
+ if not hs2_port_is_open or not beeswax_port_is_open:
raise RuntimeError("Unable to open client ports within {num_seconds} seconds.".format(
num_seconds=timeout_in_seconds))
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
index adf1320..0c6207c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/analytic-fns.test
@@ -27,6 +27,25 @@ where id % 4 = 0 and month != 1;
2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666746,'1','9'
2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666746,'1','9'
2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666746,'1','9'
+---- DBAPI_RESULTS: VERIFY_IS_EQUAL_SORTED
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,2,6,4.666666666666667,2009-02-01 20:13:00.541666,'1','9'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,3,6,3.666666666666667,2009-03-01 20:12:00.475000,'0','8'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
+2009,4,6,4.333333333333333,2009-04-01 16:11:00.416666,'1','9'
---- TYPES
INT, INT, BIGINT, DOUBLE, TIMESTAMP, STRING, STRING
====
@@ -589,6 +608,15 @@ from alltypes where id < 8
5,15,50.49999999999999,5,25.25,2009-01-01 00:07:00.210000038
6,18,70.69999999999999,6,35.34999999999999,NULL
7,13,90.89999999999999,6.5,45.45,NULL
+---- DBAPI_RESULTS: VERIFY_IS_EQUAL_SORTED
+0,1,NULL,0.5,NULL,2009-01-01 00:02:30.199999
+1,3,NULL,1,NULL,2009-01-01 00:03:30.449999
+2,6,0,2,0,2009-01-01 00:04:30.349999
+3,9,10.1,3,5.05,2009-01-01 00:05:30.124999
+4,12,30.3,4,15.15,2009-01-01 00:06:30.180000
+5,15,50.49999999999999,5,25.25,2009-01-01 00:07:00.210000
+6,18,70.69999999999999,6,35.34999999999999,NULL
+7,13,90.89999999999999,6.5,45.45,NULL
---- TYPES
INT, BIGINT, DOUBLE, DOUBLE, DOUBLE, TIMESTAMP
====
@@ -1562,6 +1590,8 @@ from (
) t3
---- RESULTS
2010-01-10 18:02:05.234931468,2010-01-10 18:02:05.215156078
+---- DBAPI_RESULTS
+2010-01-10 18:02:05.234931,2010-01-10 18:02:05.215156
---- TYPES
TIMESTAMP, TIMESTAMP
====
@@ -1848,6 +1878,13 @@ INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, TIMESTAMP
2,true,2,2,2,20,2.200000047683716,20.2,'2',2010-01-01 00:02:00.100000000
3,false,3,3,3,30,3.299999952316284,30.3,'3',2010-01-01 00:03:00.300000000
4,true,4,4,4,40,4.400000095367432,40.4,'4',2010-01-01 00:04:00.600000000
+---- DBAPI_RESULTS: VERIFY_IS_EQUAL_SORTED
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'0',2010-01-01 00:00:00
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'0',2010-01-01 00:00:00
+1,false,1,1,1,10,1.100000023841858,10.1,'1',2010-01-01 00:01:00
+2,true,2,2,2,20,2.200000047683716,20.2,'2',2010-01-01 00:02:00.100000
+3,false,3,3,3,30,3.299999952316284,30.3,'3',2010-01-01 00:03:00.300000
+4,true,4,4,4,40,4.400000095367432,40.4,'4',2010-01-01 00:04:00.600000
====
---- QUERY
# Test all possible types for first_value (except decimal).
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test b/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
index 2ed822d..33938b2 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/chars-formats.test
@@ -3,6 +3,8 @@
select * from chars_formats order by vc;
---- TYPES
char, char, string
+---- HS2_TYPES
+char, char, varchar
---- RESULTS
'abc ','8d3fffddf79e9a232ffd19f9ccaa4d6b37a6a243dbe0f23137b108a043d9da13121a9b505c804956b22e93c7f93969f4a7ba8ddea45bf4aab0bebc8f814e09918d3fffddf79e','abc'
'abcde','68f8c4575da360c32abb46689e58193a0eeaa905ae6f4a5e6c702a6ae1db35a6f86f8222b7a5489d96eb0466c755b677a64160d074617096a8c6279038bc720468f8c4575da3','b2fe9d4638503a57f93396098f24103a'
@@ -30,6 +32,8 @@ char
select vc from chars_formats order by vc;
---- TYPES
string
+---- HS2_TYPES
+varchar
---- RESULTS
'abc'
'b2fe9d4638503a57f93396098f24103a'
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/chars.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/chars.test b/testdata/workloads/functional-query/queries/QueryTest/chars.test
index cd915ce..6802647 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/chars.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/chars.test
@@ -15,6 +15,8 @@ bigint,bigint,bigint,bigint
select * from chars_tiny where cs = cast('6a' as CHAR(2))
---- TYPES
char,char,string
+---- HS2_TYPES
+char,char,varchar
---- RESULTS
'6a ','6b ','6c'
'6a ','6b ','6c'
@@ -75,6 +77,8 @@ char
select A.vc from chars_tiny as A join chars_tiny using (vc) order by A.vc
---- TYPES
string
+---- HS2_TYPES
+varchar
---- RESULTS
'1cccc'
'2cccccc'
@@ -109,6 +113,8 @@ string, string, bigint, bigint, string, string
select t1.vc, COUNT(1) FROM chars_tiny t1 GROUP BY 1 ORDER BY t1.vc
---- TYPES
string, bigint
+---- HS2_TYPES
+varchar, bigint
---- RESULTS
'1cccc',1
'2cccccc',1
@@ -151,3 +157,43 @@ char, string
'a ','a '
'NULL','NULL'
====
+---- QUERY
+# Test returning mix of nulls and non-nulls.
+WITH numbered AS (
+ SELECT *, row_number() over (order by cs) as rn
+ FROM chars_tiny)
+SELECT *
+FROM (
+ SELECT CASE WHEN rn % 2 = 0 THEN cs END cs,
+ CASE WHEN rn % 2 = 1 THEN cl END cl,
+ CASE WHEN rn % 3 = 0 THEN vc END vc
+ FROM numbered
+ UNION ALL
+ SELECT CASE WHEN rn % 2 = 1 THEN cs END cs,
+ CASE WHEN rn % 2 = 0 THEN cl END cl,
+ CASE WHEN rn % 3 = 1 THEN vc END vc
+ FROM numbered) v
+---- TYPES
+char, char, string
+---- HS2_TYPES
+char, char, varchar
+---- RESULTS
+'NULL','1bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb','NULL'
+'2aaaa','NULL','NULL'
+'NULL','3bbbbb ','3ccc'
+'4aa ','NULL','NULL'
+'NULL','5bbb ','NULL'
+'6a ','NULL','6c'
+'NULL','6b ','NULL'
+'a ','NULL','NULL'
+'NULL','NULL','NULL'
+'1aaaa','NULL','1cccc'
+'NULL','2bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb','NULL'
+'3aaa ','NULL','NULL'
+'NULL','4bbbb ','4cc'
+'5a ','NULL','NULL'
+'NULL','6b ','NULL'
+'6a ','NULL','6c'
+'NULL','b ','NULL'
+'NULL','NULL','NULL'
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test b/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
index e88d928..62ba37b 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/data-source-tables.test
@@ -13,6 +13,12 @@ where float_col != 0 and
1992,true,2,92,1992,19920,2191.199951171875,1992,1970-01-01 00:00:01.992000000,'1992',-999998007,-9999998007,-9999999999.9999998007,-9.9999999999999999999999999999999998007,-99999.98007
1993,false,3,93,1993,19930,2192.300048828125,1993,1970-01-01 00:00:01.993000000,'1993',999998006,9999998006,9999999999.9999998006,9.9999999999999999999999999999999998006,99999.98006
1994,true,4,94,1994,19940,2193.39990234375,1994,1970-01-01 00:00:01.994000000,'1994',-999998005,-9999998005,-9999999999.9999998005,-9.9999999999999999999999999999999998005,-99999.98005
+---- DBAPI_RESULTS
+1990,true,0,90,1990,19900,2189,1990,1970-01-01 00:00:01.990000,'NULL',-999998009,-9999998009,-9999999999.9999998009,-9.9999999999999999999999999999999998009,-99999.98009
+1991,false,1,91,1991,19910,2190.10009765625,1991,1970-01-01 00:00:01.991000,'1991',999998008,9999998008,9999999999.9999998008,9.9999999999999999999999999999999998008,99999.98008
+1992,true,2,92,1992,19920,2191.199951171875,1992,1970-01-01 00:00:01.992000,'1992',-999998007,-9999998007,-9999999999.9999998007,-9.9999999999999999999999999999999998007,-99999.98007
+1993,false,3,93,1993,19930,2192.300048828125,1993,1970-01-01 00:00:01.993000,'1993',999998006,9999998006,9999999999.9999998006,9.9999999999999999999999999999999998006,99999.98006
+1994,true,4,94,1994,19940,2193.39990234375,1994,1970-01-01 00:00:01.994000,'1994',-999998005,-9999998005,-9999999999.9999998005,-9.9999999999999999999999999999999998005,-99999.98005
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, TIMESTAMP, STRING, DECIMAL, DECIMAL, DECIMAL, DECIMAL, DECIMAL
====
@@ -25,6 +31,10 @@ where double_col != 0 and int_col >= 1990 limit 3
19900,1970-01-01 00:00:01.990000000,1990
19910,1970-01-01 00:00:01.991000000,1991
19920,1970-01-01 00:00:01.992000000,1992
+---- DBAPI_RESULTS
+19900,1970-01-01 00:00:01.990000,1990
+19910,1970-01-01 00:00:01.991000,1991
+19920,1970-01-01 00:00:01.992000,1992
---- TYPES
BIGINT, TIMESTAMP, DOUBLE
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/decimal.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/decimal.test b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
index ac7f1c1..1e1affc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/decimal.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/decimal.test
@@ -385,3 +385,36 @@ NULL,0.0000
---- TYPES
DECIMAL, DECIMAL
====
+---- QUERY
+# Query return mixed NULL and non-NULL decimal values.
+select case when rn % 2 = 0 then c1 end,
+ case when rn % 3 = 0 then c2 end,
+ case when rn % 4 = 0 then c3 end
+from (
+ select *, row_number() over (order by c1) rn
+ from decimal_tiny) v
+where rn <= 20
+---- RESULTS
+NULL,NULL,NULL
+0.1111,NULL,NULL
+NULL,102.44444,NULL
+0.3333,NULL,0.3
+NULL,NULL,NULL
+0.5555,106.11110,NULL
+NULL,NULL,NULL
+0.7777,NULL,0.7
+NULL,109.77776,NULL
+0.9999,NULL,NULL
+NULL,NULL,NULL
+1.2221,113.44442,0.1
+NULL,NULL,NULL
+1.4443,NULL,NULL
+NULL,117.11108,NULL
+1.6665,NULL,0.5
+NULL,NULL,NULL
+1.8887,120.77774,NULL
+NULL,NULL,NULL
+2.1109,NULL,0.9
+---- TYPES
+DECIMAL, DECIMAL, DECIMAL
+====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test b/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
index 2c3a72b..96e2fcc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/inline-view-limit.test
@@ -21,6 +21,12 @@ INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIM
4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000000,2009,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000000,2009,1
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
+6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000,2009,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000,2009,1
====
---- QUERY
# order by/limit in subquery, followed by addititional selection predicate;
@@ -39,6 +45,12 @@ INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIM
4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000000,2009,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000000,2009,1
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
+6,true,6,6,6,60,6.599999904632568,60.6,'01/01/09','6',2009-01-01 00:06:00.150000,2009,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/09','8',2009-01-01 00:08:00.280000,2009,1
====
---- QUERY
# join against subquery with limit creates a merge fragment that applies the limit
@@ -102,6 +114,12 @@ double_col, date_string_col, string_col, timestamp_col, year, month from
2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000000,2009,1
3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000000,2009,1
4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000,2009,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP, INT, INT
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
index dd0094b..49b2e91 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/inline-view.test
@@ -151,6 +151,12 @@ where month = 1
2,3,2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000000,2009,1
3,3,3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000000,2009,1
4,3,4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000000,2009,1
+---- DBAPI_RESULTS
+0,3,0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,3,1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,3,2,true,2,2,2,20,2.200000047683716,20.2,'01/01/09','2',2009-01-01 00:02:00.100000,2009,1
+3,3,3,false,3,3,3,30,3.299999952316284,30.3,'01/01/09','3',2009-01-01 00:03:00.300000,2009,1
+4,3,4,true,4,4,4,40,4.400000095367432,40.4,'01/01/09','4',2009-01-01 00:04:00.600000,2009,1
---- TYPES
int, bigint, int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/limit.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/limit.test b/testdata/workloads/functional-query/queries/QueryTest/limit.test
index 334ed45..49c6129 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/limit.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/limit.test
@@ -46,6 +46,17 @@ select * from (select * from alltypesagg where day = 1 order by id limit 10) a l
7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000000,2010,1,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
+---- DBAPI_RESULTS : VERIFY_IS_EQUAL_SORTED
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632,60.6,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -63,6 +74,17 @@ select * from alltypesagg where day = 1 order by id limit 10
7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000000,2010,1,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
+---- DBAPI_RESULTS
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+1,false,1,1,1,10,1.100000023841858,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316284,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632568,60.6,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530273,90.90000000000001,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -80,6 +102,17 @@ select * from alltypesagg where tinyint_col is null and day = 1 order by id limi
70,true,NULL,70,70,700,77,707,'01/01/10','70',2010-01-01 01:10:24.150000000,2010,1,1
80,true,NULL,80,80,800,88,808,'01/01/10','80',2010-01-01 01:20:31.600000000,2010,1,1
90,true,NULL,90,90,900,99,909,'01/01/10','90',2010-01-01 01:30:40.500000000,2010,1,1
+---- DBAPI_RESULTS
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+10,true,NULL,10,10,100,11,101,'01/01/10','10',2010-01-01 00:10:00.450000,2010,1,1
+20,true,NULL,20,20,200,22,202,'01/01/10','20',2010-01-01 00:20:01.900000,2010,1,1
+30,true,NULL,30,30,300,33,303,'01/01/10','30',2010-01-01 00:30:04.350000,2010,1,1
+40,true,NULL,40,40,400,44,404,'01/01/10','40',2010-01-01 00:40:07.800000,2010,1,1
+50,true,NULL,50,50,500,55,505,'01/01/10','50',2010-01-01 00:50:12.250000,2010,1,1
+60,true,NULL,60,60,600,66,606,'01/01/10','60',2010-01-01 01:00:17.700000,2010,1,1
+70,true,NULL,70,70,700,77,707,'01/01/10','70',2010-01-01 01:10:24.150000,2010,1,1
+80,true,NULL,80,80,800,88,808,'01/01/10','80',2010-01-01 01:20:31.600000,2010,1,1
+90,true,NULL,90,90,900,99,909,'01/01/10','90',2010-01-01 01:30:40.500000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -247,6 +280,17 @@ select * from alltypesagg where id != 0 and id != 10 order by id limit 10 offset
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
11,false,1,11,11,110,12.10000038146973,111.1,'01/01/10','11',2010-01-01 00:11:00.550000000,2010,1,1
+---- DBAPI_RESULTS
+1,false,1,1,1,10,1.100000023841858,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683716,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316284,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367432,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632568,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
+11,false,1,11,11,110,12.10000038146973,111.1,'01/01/10','11',2010-01-01 00:11:00.550000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -264,6 +308,17 @@ select * from alltypesagg where id != 0 and id != 10 order by id limit 10 offset
14,true,4,14,14,140,15.39999961853027,141.4,'01/01/10','14',2010-01-01 00:14:00.910000000,2010,1,1
15,false,5,15,15,150,16.5,151.5,'01/01/10','15',2010-01-01 00:15:01.500000000,2010,1,1
16,true,6,16,16,160,17.60000038146973,161.6,'01/01/10','16',2010-01-01 00:16:01.200000000,2010,1,1
+---- DBAPI_RESULTS
+6,true,6,6,6,60,6.599999904632568,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
+11,false,1,11,11,110,12.10000038146973,111.1,'01/01/10','11',2010-01-01 00:11:00.550000,2010,1,1
+12,true,2,12,12,120,13.19999980926514,121.2,'01/01/10','12',2010-01-01 00:12:00.660000,2010,1,1
+13,false,3,13,13,130,14.30000019073486,131.3,'01/01/10','13',2010-01-01 00:13:00.780000,2010,1,1
+14,true,4,14,14,140,15.39999961853027,141.4,'01/01/10','14',2010-01-01 00:14:00.910000,2010,1,1
+15,false,5,15,15,150,16.5,151.5,'01/01/10','15',2010-01-01 00:15:01.500000,2010,1,1
+16,true,6,16,16,160,17.60000038146973,161.6,'01/01/10','16',2010-01-01 00:16:01.200000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/top-n.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/top-n.test b/testdata/workloads/functional-query/queries/QueryTest/top-n.test
index 7d52731..32b1504 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/top-n.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/top-n.test
@@ -736,6 +736,27 @@ from alltypessmall order by 1 limit 20
2008-12-20 00:19:00.810000000,2009-02-02 00:19:00.810000000,44
2008-12-20 00:20:00.900000000,2009-02-03 00:20:00.900000000,45
2008-12-20 00:23:00.930000000,2009-03-03 00:23:00.930000000,73
+---- DBAPI_RESULTS
+2008-12-10 00:24:00.960000,2009-01-03 00:24:00.960000,24
+2008-12-11 00:23:00.930000,2009-01-03 00:23:00.930000,23
+2008-12-12 00:22:00.910000,2009-01-03 00:22:00.910000,22
+2008-12-13 00:21:00.900000,2009-01-03 00:21:00.900000,21
+2008-12-14 00:19:00.810000,2009-01-02 00:19:00.810000,19
+2008-12-14 00:20:00.900000,2009-01-03 00:20:00.900000,20
+2008-12-15 00:18:00.730000,2009-01-02 00:18:00.730000,18
+2008-12-16 00:17:00.660000,2009-01-02 00:17:00.660000,17
+2008-12-16 00:24:00.960000,2009-02-03 00:24:00.960000,49
+2008-12-17 00:16:00.600000,2009-01-02 00:16:00.600000,16
+2008-12-17 00:23:00.930000,2009-02-03 00:23:00.930000,48
+2008-12-18 00:15:00.550000,2009-01-02 00:15:00.550000,15
+2008-12-18 00:22:00.910000,2009-02-03 00:22:00.910000,47
+2008-12-19 00:14:00.510000,2009-01-02 00:14:00.510000,14
+2008-12-19 00:21:00.900000,2009-02-03 00:21:00.900000,46
+2008-12-19 00:24:00.960000,2009-03-03 00:24:00.960000,74
+2008-12-20 00:13:00.480000,2009-01-02 00:13:00.480000,13
+2008-12-20 00:19:00.810000,2009-02-02 00:19:00.810000,44
+2008-12-20 00:20:00.900000,2009-02-03 00:20:00.900000,45
+2008-12-20 00:23:00.930000,2009-03-03 00:23:00.930000,73
---- TYPES
TIMESTAMP, TIMESTAMP, INT
====
@@ -816,6 +837,17 @@ select * from alltypesagg where day is not null order by id limit 10 offset 0
7,false,7,7,7,70,7.699999809265137,70.7,'01/01/10','7',2010-01-01 00:07:00.210000000,2010,1,1
8,true,8,8,8,80,8.800000190734863,80.8,'01/01/10','8',2010-01-01 00:08:00.280000000,2010,1,1
9,false,9,9,9,90,9.899999618530273,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000000,2010,1,1
+---- DBAPI_RESULTS
+0,true,NULL,NULL,NULL,NULL,NULL,NULL,'01/01/10','0',2010-01-01 00:00:00,2010,1,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/10','1',2010-01-01 00:01:00,2010,1,1
+2,true,2,2,2,20,2.200000047683,20.2,'01/01/10','2',2010-01-01 00:02:00.100000,2010,1,1
+3,false,3,3,3,30,3.299999952316,30.3,'01/01/10','3',2010-01-01 00:03:00.300000,2010,1,1
+4,true,4,4,4,40,4.400000095367,40.4,'01/01/10','4',2010-01-01 00:04:00.600000,2010,1,1
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -834,6 +866,17 @@ select * from alltypesagg where day is not null order by id limit 10 offset 5
12,true,2,12,12,120,13.19999980926514,121.2,'01/01/10','12',2010-01-01 00:12:00.660000000,2010,1,1
13,false,3,13,13,130,14.30000019073486,131.3,'01/01/10','13',2010-01-01 00:13:00.780000000,2010,1,1
14,true,4,14,14,140,15.39999961853027,141.4,'01/01/10','14',2010-01-01 00:14:00.910000000,2010,1,1
+---- DBAPI_RESULTS
+5,false,5,5,5,50,5.5,50.5,'01/01/10','5',2010-01-01 00:05:00.100000,2010,1,1
+6,true,6,6,6,60,6.599999904632,60.59999999999999,'01/01/10','6',2010-01-01 00:06:00.150000,2010,1,1
+7,false,7,7,7,70,7.699999809265,70.7,'01/01/10','7',2010-01-01 00:07:00.210000,2010,1,1
+8,true,8,8,8,80,8.800000190734,80.8,'01/01/10','8',2010-01-01 00:08:00.280000,2010,1,1
+9,false,9,9,9,90,9.899999618530,90.89999999999999,'01/01/10','9',2010-01-01 00:09:00.360000,2010,1,1
+10,true,NULL,10,10,100,11,101,'01/01/10','10',2010-01-01 00:10:00.450000,2010,1,1
+11,false,1,11,11,110,12.10000038146,111.1,'01/01/10','11',2010-01-01 00:11:00.550000,2010,1,1
+12,true,2,12,12,120,13.19999980926,121.2,'01/01/10','12',2010-01-01 00:12:00.660000,2010,1,1
+13,false,3,13,13,130,14.30000019073,131.3,'01/01/10','13',2010-01-01 00:13:00.780000,2010,1,1
+14,true,4,14,14,140,15.39999961853,141.4,'01/01/10','14',2010-01-01 00:14:00.910000,2010,1,1
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int, int
====
@@ -901,6 +944,9 @@ limit 6
20,NULL,NULL
---- TYPES
int, tinyint, null
+---- HS2_TYPES
+# HS2 maps NULL to BOOLEAN
+int, tinyint, boolean
====
---- QUERY
# check cross joins within a subquery
@@ -963,6 +1009,27 @@ limit 100
83,83,3,3,983,8.800000190734863,'3',1
91,91,1,1,991,6.599999904632568,'1',0
91,91,1,1,991,6.599999904632568,'1',1
+---- DBAPI_RESULTS
+31,31,1,1,931,6.599999904632,'1',0
+31,31,1,1,931,6.599999904632,'1',1
+32,32,2,2,932,7.699999809265,'2',0
+32,32,2,2,932,7.699999809265,'2',1
+33,33,3,3,933,8.800000190734,'3',0
+33,33,3,3,933,8.800000190734,'3',1
+41,41,1,1,941,6.599999904632,'1',0
+41,41,1,1,941,6.599999904632,'1',1
+42,42,2,2,942,7.699999809265,'2',0
+42,42,2,2,942,7.699999809265,'2',1
+43,43,3,3,943,8.800000190734,'3',0
+43,43,3,3,943,8.800000190734,'3',1
+81,81,1,1,981,6.599999904632,'1',0
+81,81,1,1,981,6.599999904632,'1',1
+82,82,2,2,982,7.699999809265,'2',0
+82,82,2,2,982,7.699999809265,'2',1
+83,83,3,3,983,8.800000190734,'3',0
+83,83,3,3,983,8.800000190734,'3',1
+91,91,1,1,991,6.599999904632,'1',0
+91,91,1,1,991,6.599999904632,'1',1
---- TYPES
SMALLINT, INT, TINYINT, INT, INT, FLOAT, STRING, INT
====
@@ -1006,6 +1073,10 @@ order by 1 limit 3
0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1024,6 +1095,10 @@ order by 1,2 limit 3
0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1043,6 +1118,11 @@ order by 1,2 limit 4
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1069,6 +1149,14 @@ limit 20
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+3,false,1,1,1,10,1.10000002384158,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1093,6 +1181,12 @@ limit 20
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
3,false,1,1,1,10,1.100000023841858,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+3,false,1,1,1,10,1.100000023841,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1132,6 +1226,18 @@ limit 20
6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
7,false,1,1,1,10,1.100000023841858,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
7,false,1,1,1,10,1.100000023841858,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+3,false,1,1,1,10,1.100000023841,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+5,false,1,1,1,10,1.100000023841,10.1,'03/01/09','1',2009-03-01 00:01:00,2009,3
+6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
+6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
+7,false,1,1,1,10,1.100000023841,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
+7,false,1,1,1,10,1.100000023841,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1151,6 +1257,12 @@ order by 1 limit 5
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1168,6 +1280,10 @@ order by 1 limit 3
0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
1,false,1,1,1,10,1.100000023841858,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+---- DBAPI_RESULTS
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
---- TYPES
int, boolean, tinyint, smallint, int, bigint, float, double, string, string, timestamp, int, int
====
@@ -1243,6 +1359,15 @@ select * from functional.alltypestiny order by 2,1 limit 10
2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
+---- DBAPI_RESULTS
+1,false,1,1,1,10,1.100000023841,10.1,'01/01/09','1',2009-01-01 00:01:00,2009,1
+3,false,1,1,1,10,1.100000023841,10.1,'02/01/09','1',2009-02-01 00:01:00,2009,2
+5,false,1,1,1,10,1.100000023841,10.1,'03/01/09','1',2009-03-01 00:01:00,2009,3
+7,false,1,1,1,10,1.100000023841,10.1,'04/01/09','1',2009-04-01 00:01:00,2009,4
+0,true,0,0,0,0,0,0,'01/01/09','0',2009-01-01 00:00:00,2009,1
+2,true,0,0,0,0,0,0,'02/01/09','0',2009-02-01 00:00:00,2009,2
+4,true,0,0,0,0,0,0,'03/01/09','0',2009-03-01 00:00:00,2009,3
+6,true,0,0,0,0,0,0,'04/01/09','0',2009-04-01 00:00:00,2009,4
---- TYPES
INT, BOOLEAN, TINYINT, SMALLINT, INT, BIGINT, FLOAT, DOUBLE, STRING, STRING, TIMESTAMP, INT, INT
====
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/testdata/workloads/functional-query/queries/QueryTest/union.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/union.test b/testdata/workloads/functional-query/queries/QueryTest/union.test
index 7fe4020..6448fc7 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/union.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/union.test
@@ -740,6 +740,8 @@ union all
select 3, 'c', NULL, 30.0
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
@@ -754,6 +756,8 @@ union distinct
select 1, 'a', NULL, 10.0
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
@@ -767,6 +771,8 @@ union all
values(3, 'c', NULL, 30.0)
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
@@ -781,6 +787,8 @@ union distinct
values(1, 'a', NULL, 10.0)
---- TYPES
tinyint, string, null, decimal
+---- HS2_TYPES
+tinyint, string, boolean, decimal
---- RESULTS: VERIFY_IS_EQUAL_SORTED
1,'a',NULL,10.0
2,'b',NULL,20.0
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/beeswax/impala_beeswax.py
----------------------------------------------------------------------
diff --git a/tests/beeswax/impala_beeswax.py b/tests/beeswax/impala_beeswax.py
index 1a3bc65..d8272ed 100644
--- a/tests/beeswax/impala_beeswax.py
+++ b/tests/beeswax/impala_beeswax.py
@@ -74,6 +74,12 @@ class ImpalaBeeswaxResult(object):
self.time_taken = kwargs.get('time_taken', 0)
self.summary = kwargs.get('summary', str())
self.schema = kwargs.get('schema', None)
+ self.column_types = None
+ self.column_labels = None
+ if self.schema is not None:
+ # Extract labels and types so there is a shared interface with HS2ResultSet.
+ self.column_types = [fs.type.upper() for fs in self.schema.fieldSchemas]
+ self.column_labels = [fs.name.upper() for fs in self.schema.fieldSchemas]
self.runtime_profile = kwargs.get('runtime_profile', str())
self.exec_summary = kwargs.get('exec_summary', None)
@@ -420,14 +426,6 @@ class ImpalaBeeswaxClient(object):
def get_log(self, query_handle):
return self.__do_rpc(lambda: self.imp_service.get_log(query_handle))
- def refresh(self):
- """Invalidate the Impalad catalog"""
- return self.execute("invalidate metadata")
-
- def refresh_table(self, db_name, table_name):
- """Refresh a specific table from the catalog"""
- return self.execute("refresh %s.%s" % (db_name, table_name))
-
def fetch_results(self, query_string, query_handle, max_rows = -1):
"""Fetches query results given a handle and query type (insert, use, other)"""
query_type = self.__get_query_type(query_string)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/impala_connection.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_connection.py b/tests/common/impala_connection.py
index b075506..e7dace9 100644
--- a/tests/common/impala_connection.py
+++ b/tests/common/impala_connection.py
@@ -21,7 +21,9 @@
import abc
import logging
+import re
+import impala.dbapi as impyla
from tests.beeswax.impala_beeswax import ImpalaBeeswaxClient
@@ -33,32 +35,37 @@ console_handler.setFormatter(logging.Formatter('%(message)s'))
LOG.addHandler(console_handler)
LOG.propagate = False
+# Regular expression that matches the "progress" entry in the HS2 log.
+PROGRESS_LOG_RE = re.compile(
+ r'^Query [a-z0-9:]+ [0-9]+% Complete \([0-9]+ out of [0-9]+\)$')
+
# Common wrapper around the internal types of HS2/Beeswax operation/query handles.
class OperationHandle(object):
- def __init__(self, handle):
+ def __init__(self, handle, sql_stmt):
self.__handle = handle
+ self.__sql_stmt = sql_stmt
+
+ def get_handle(self):
+ return self.__handle
- def get_handle(self): return self.__handle
+ def sql_stmt(self):
+ return self.__sql_stmt
# Represents an Impala connection.
class ImpalaConnection(object):
__metaclass__ = abc.ABCMeta
-
@abc.abstractmethod
def set_configuration_option(self, name, value):
"""Sets a configuraiton option name to the given value"""
pass
- @abc.abstractmethod
- def get_configuration(self):
- """Returns the configuration (a dictionary of key-value pairs) for this connection"""
- pass
-
- @abc.abstractmethod
- def set_configuration(self, configuration_option_dict):
+ def set_configuration(self, config_option_dict):
"""Replaces existing configuration with the given dictionary"""
- pass
+ assert config_option_dict is not None, "config_option_dict cannot be None"
+ self.clear_configuration()
+ for name, value in config_option_dict.iteritems():
+ self.set_configuration_option(name, value)
@abc.abstractmethod
def clear_configuration(self):
@@ -66,6 +73,13 @@ class ImpalaConnection(object):
pass
@abc.abstractmethod
+ def get_default_configuration(self):
+ """Return the default configuration for the connection, before any modifications are
+ made to the session state. Returns a map with the config variable as the key and a
+ string representation of the default value as the value."""
+ pass
+
+ @abc.abstractmethod
def connect(self):
"""Opens the connection"""
pass
@@ -87,7 +101,7 @@ class ImpalaConnection(object):
@abc.abstractmethod
def get_log(self, operation_handle):
- """Returns the log of an operation"""
+ """Returns the log of an operation as a string, with entries separated by newlines."""
pass
@abc.abstractmethod
@@ -101,7 +115,9 @@ class ImpalaConnection(object):
@abc.abstractmethod
def execute_async(self, sql_stmt):
- """Issues a query and returns the handle to the caller for processing"""
+ """Issues a query and returns the handle to the caller for processing. Only one
+ async operation per connection at a time is supported, due to limitations of the
+ Beeswax protocol and the Impyla client."""
pass
@abc.abstractmethod
@@ -127,17 +143,11 @@ class BeeswaxConnection(ImpalaConnection):
LOG.info('SET %s=%s;' % (name, value))
self.__beeswax_client.set_query_option(name, value)
- def get_configuration(self):
- return self.__beeswax_client.get_query_options()
-
def get_default_configuration(self):
- return self.__beeswax_client.get_default_configuration()
-
- def set_configuration(self, config_option_dict):
- assert config_option_dict is not None, "config_option_dict cannot be None"
- self.clear_configuration()
- for name, value in config_option_dict.iteritems():
- self.set_configuration_option(name, value)
+ result = {}
+ for item in self.__beeswax_client.get_default_configuration():
+ result[item.key] = item.value
+ return result
def clear_configuration(self):
self.__beeswax_client.clear_query_options()
@@ -161,7 +171,8 @@ class BeeswaxConnection(ImpalaConnection):
def execute_async(self, sql_stmt, user=None):
LOG.info("-- executing async: %s\n%s;\n" % (self.__host_port, sql_stmt))
- return OperationHandle(self.__beeswax_client.execute_query_async(sql_stmt, user=user))
+ beeswax_handle = self.__beeswax_client.execute_query_async(sql_stmt, user=user)
+ return OperationHandle(beeswax_handle, sql_stmt)
def cancel(self, operation_handle):
LOG.info("-- canceling operation: %s" % operation_handle)
@@ -197,26 +208,184 @@ class BeeswaxConnection(ImpalaConnection):
LOG.info("-- getting log for operation: %s" % operation_handle)
return self.__beeswax_client.get_log(operation_handle.get_handle())
- def refresh(self):
- """Invalidate the Impalad catalog"""
- return self.execute("invalidate metadata")
-
- def invalidate_table(self, table_name):
- """Invalidate a specific table from the catalog"""
- return self.execute("invalidate metadata %s" % (table_name))
-
- def refresh_table(self, db_name, table_name):
- """Refresh a specific table from the catalog"""
- return self.execute("refresh %s.%s" % (db_name, table_name))
-
def fetch(self, sql_stmt, operation_handle, max_rows = -1):
LOG.info("-- fetching results from: %s" % operation_handle)
return self.__beeswax_client.fetch_results(
sql_stmt, operation_handle.get_handle(), max_rows)
-def create_connection(host_port, use_kerberos=False):
- # TODO: Support HS2 connections.
- return BeeswaxConnection(host_port=host_port, use_kerberos=use_kerberos)
+
+class ImpylaHS2Connection(ImpalaConnection):
+ """Connection to Impala using the impyla client connecting to HS2 endpoint.
+ impyla implements the standard Python dbabi: https://www.python.org/dev/peps/pep-0249/
+ plus Impala-specific extensions, e.g. for fetching runtime profiles.
+ TODO: implement support for kerberos, SSL, etc.
+ """
+ def __init__(self, host_port, use_kerberos=False):
+ self.__host_port = host_port
+ if use_kerberos:
+ raise NotImplementedError("Kerberos support not yet implemented")
+ # Impyla connection and cursor is initialised in connect(). We need to reuse the same
+ # cursor for different operations (as opposed to creating a new cursor per operation)
+ # so that the session is preserved. This means that we can only execute one operation
+ # at a time per connection, which is a limitation also imposed by the Beeswax API.
+ self.__impyla_conn = None
+ self.__cursor = None
+ # Query options to send along with each query.
+ self.__query_options = {}
+
+ def set_configuration_option(self, name, value):
+ self.__query_options[name] = str(value)
+
+ def get_default_configuration(self):
+ return self.__default_query_options.copy()
+
+ def clear_configuration(self):
+ self.__query_options.clear()
+
+ def connect(self):
+ LOG.info("-- connecting to {0} with impyla".format(self.__host_port))
+ host, port = self.__host_port.split(":")
+ self.__impyla_conn = impyla.connect(host=host, port=int(port))
+ LOG.info("Conn {0}".format(self.__impyla_conn))
+ # Get the default query options for the session before any modifications are made.
+ self.__cursor = self.__impyla_conn.cursor()
+ self.__cursor.execute("set all")
+ self.__default_query_options = {}
+ for name, val, _ in self.__cursor:
+ self.__default_query_options[name] = val
+ self.__cursor.close_operation()
+ LOG.debug("Default query options: {0}".format(self.__default_query_options))
+
+ def close(self):
+ LOG.info("-- closing connection to: {0}".format(self.__host_port))
+ self.__impyla_conn.close()
+
+ def close_query(self, operation_handle):
+ LOG.info("-- closing query for operation handle: {0}".format(operation_handle))
+ operation_handle.get_handle().close_operation()
+
+ def execute(self, sql_stmt, user=None):
+ handle = self.execute_async(sql_stmt, user)
+ try:
+ return self.__fetch_results(handle)
+ finally:
+ self.close_query(handle)
+
+ def execute_async(self, sql_stmt, user=None):
+ LOG.info("-- executing: {0}\n{1};\n".format(self.__host_port, sql_stmt))
+ if user is not None:
+ raise NotImplementedError("Not yet implemented for HS2 - authentication")
+ try:
+ self.__cursor.execute(sql_stmt, configuration=self.__query_options)
+ return OperationHandle(self.__cursor, sql_stmt)
+ except Exception:
+ self.__cursor.close_operation()
+ raise
+
+ def cancel(self, operation_handle):
+ LOG.info("-- canceling operation: {0}".format(operation_handle))
+ return self.__beeswax_client.cancel_query(operation_handle.get_handle())
+
+ def get_state(self, operation_handle):
+ LOG.info("-- getting state for operation: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def get_exec_summary(self, operation_handle):
+ LOG.info("-- getting exec summary operation: {0}".format(operation_handle))
+ raise NotImplementedError(
+ "Not yet implemented for HS2 - summary returned is thrift, not string.")
+
+ def get_runtime_profile(self, operation_handle):
+ LOG.info("-- getting runtime profile operation: {0}".format(operation_handle))
+ return self.__cursor.get_profile()
+
+ def wait_for_finished_timeout(self, operation_handle, timeout):
+ LOG.info("-- waiting for query to reach FINISHED state: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def wait_for_admission_control(self, operation_handle):
+ LOG.info("-- waiting for completion of the admission control processing of the "
+ "query: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def get_admission_result(self, operation_handle):
+ LOG.info("-- getting the admission result: {0}".format(operation_handle))
+ raise NotImplementedError("Not yet implemented for HS2 - states differ from beeswax")
+
+ def get_log(self, operation_handle):
+ LOG.info("-- getting log for operation: {0}".format(operation_handle))
+ # HS2 includes non-error log messages that we need to filter out.
+ cursor = operation_handle.get_handle()
+ lines = [line for line in cursor.get_log().split('\n')
+ if not PROGRESS_LOG_RE.match(line)]
+ return '\n'.join(lines)
+
+ def fetch(self, sql_stmt, handle, max_rows=-1):
+ LOG.info("-- fetching results from: {0}".format(handle))
+ return self.__fetch_results(handle, max_rows)
+
+ def __fetch_results(self, handle, max_rows=-1):
+ """Implementation of result fetching from handle."""
+ cursor = handle.get_handle()
+ assert cursor is not None
+ # Don't fetch data for queries with no results.
+ result_tuples = None
+ column_labels = None
+ column_types = None
+ if cursor.has_result_set:
+ desc = cursor.description
+ column_labels = [col_desc[0].upper() for col_desc in desc]
+ column_types = [col_desc[1].upper() for col_desc in desc]
+ if max_rows < 0:
+ result_tuples = cursor.fetchall()
+ else:
+ result_tuples = cursor.fetchmany(max_rows)
+ log = self.get_log(handle)
+ profile = self.get_runtime_profile(handle)
+ return ImpylaHS2ResultSet(success=True, result_tuples=result_tuples,
+ column_labels=column_labels, column_types=column_types,
+ query=handle.sql_stmt(), log=log, profile=profile)
+
+
+class ImpylaHS2ResultSet(object):
+ """This emulates the interface of ImpalaBeeswaxResult so that it can be used in
+ place of it. TODO: when we deprecate/remove Beeswax, clean this up."""
+ def __init__(self, success, result_tuples, column_labels, column_types, query, log,
+ profile):
+ self.success = success
+ self.column_labels = column_labels
+ self.column_types = column_types
+ self.query = query
+ self.log = log
+ self.profile = profile
+ self.__result_tuples = result_tuples
+ # self.data is the data in the ImpalaBeeswaxResult format: a list of rows with each
+ # row represented as a tab-separated string.
+ self.data = None
+ if result_tuples is not None:
+ self.data = [self.__convert_result_row(tuple) for tuple in result_tuples]
+
+ def __convert_result_row(self, result_tuple):
+ """Take primitive values from a result tuple and construct the tab-separated string
+ that would have been returned via beeswax."""
+ return '\t'.join([self.__convert_result_value(val, type)
+ for val, type in zip(result_tuple, self.column_types)])
+
+ def __convert_result_value(self, val, type):
+ """Take a primitive value from a result tuple and its type and construct the string
+ that would have been returned via beeswax."""
+ if val is None:
+ return 'NULL'
+ else:
+ return str(val)
+
+
+def create_connection(host_port, use_kerberos=False, protocol='beeswax'):
+ if protocol == 'beeswax':
+ return BeeswaxConnection(host_port=host_port, use_kerberos=use_kerberos)
+ else:
+ assert protocol == 'hs2'
+ return ImpylaHS2Connection(host_port=host_port, use_kerberos=use_kerberos)
def create_ldap_connection(host_port, user, password, use_ssl=False):
return BeeswaxConnection(host_port=host_port, user=user, password=password,
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/impala_service.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 0ad4496..16419ae 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -283,10 +283,21 @@ class ImpaladService(BaseImpalaService):
def create_beeswax_client(self, use_kerberos=False):
"""Creates a new beeswax client connection to the impalad"""
- client = create_connection('%s:%d' % (self.hostname, self.beeswax_port), use_kerberos)
+ client = create_connection('%s:%d' % (self.hostname, self.beeswax_port),
+ use_kerberos, 'beeswax')
client.connect()
return client
+ def beeswax_port_is_open(self):
+ """Test if the beeswax port is open. Does not need to authenticate."""
+ try:
+ # The beeswax client will connect successfully even if not authenticated.
+ client = self.create_beeswax_client()
+ client.close()
+ return True
+ except Exception:
+ return False
+
def create_ldap_beeswax_client(self, user, password, use_ssl=False):
client = create_ldap_connection('%s:%d' % (self.hostname, self.beeswax_port),
user=user, password=password, use_ssl=use_ssl)
@@ -295,14 +306,23 @@ class ImpaladService(BaseImpalaService):
def create_hs2_client(self):
"""Creates a new HS2 client connection to the impalad"""
- host, port = (self.hostname, self.hs2_port)
- socket = TSocket(host, port)
- transport = TBufferedTransport(socket)
- transport.open()
- protocol = TBinaryProtocol.TBinaryProtocol(transport)
- hs2_client = TCLIService.Client(protocol)
- return hs2_client
+ client = create_connection('%s:%d' % (self.hostname, self.hs2_port), protocol='hs2')
+ client.connect()
+ return client
+ def hs2_port_is_open(self):
+ """Test if the HS2 port is open. Does not need to authenticate."""
+ # Impyla will try to authenticate as part of connecting, so preserve previous logic
+ # that uses the HS2 thrift code directly.
+ try:
+ socket = TSocket(self.hostname, self.hs2_port)
+ transport = TBufferedTransport(socket)
+ transport.open()
+ transport.close()
+ return True
+ except Exception, e:
+ LOG.info(e)
+ return False
# Allows for interacting with the StateStore service to perform operations such as
# accessing the debug webpage.
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/impala_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/impala_test_suite.py b/tests/common/impala_test_suite.py
index 0f18dea..954ca4c 100644
--- a/tests/common/impala_test_suite.py
+++ b/tests/common/impala_test_suite.py
@@ -122,11 +122,14 @@ class ImpalaTestSuite(BaseTestSuite):
cls.ImpalaTestMatrix.add_dimension(
cls.create_table_info_dimension(cls.exploration_strategy()))
cls.ImpalaTestMatrix.add_dimension(cls.__create_exec_option_dimension())
+ # Execute tests through Beeswax by default. Individual tests that have been converted
+ # to work with the HS2 client can add HS2 in addition to or instead of beeswax.
+ cls.ImpalaTestMatrix.add_dimension(ImpalaTestDimension('protocol', 'beeswax'))
@classmethod
def setup_class(cls):
"""Setup section that runs before each test suite"""
- cls.hive_client, cls.client = [None, None]
+ cls.hive_client, cls.client, cls.hs2_client = [None, None, None]
# Create a Hive Metastore Client (used for executing some test SETUP steps
metastore_host, metastore_port = pytest.config.option.metastore_server.split(':')
trans_type = 'buffered'
@@ -141,8 +144,15 @@ class ImpalaTestSuite(BaseTestSuite):
cls.hive_client = ThriftHiveMetastore.Client(protocol)
cls.hive_transport.open()
- # Create a connection to Impala.
- cls.client = cls.create_impala_client(IMPALAD)
+ # Create a connection to Impala, self.client is Beeswax so that existing tests that
+ # assume beeswax do not need modification (yet).
+ cls.client = cls.create_impala_client(protocol='beeswax')
+ try:
+ cls.hs2_client = cls.create_impala_client(protocol='hs2')
+ except Exception, e:
+ # HS2 connection can fail for benign reasons, e.g. running with unsupported auth.
+ LOG.info("HS2 connection setup failed, continuing...: {0}", e)
+
# Default query options are populated on demand.
cls.default_query_options = {}
@@ -165,18 +175,42 @@ class ImpalaTestSuite(BaseTestSuite):
# Cleanup the Impala and Hive Metastore client connections
if cls.hive_transport:
cls.hive_transport.close()
-
if cls.client:
cls.client.close()
+ if cls.hs2_client:
+ cls.hs2_client.close()
@classmethod
- def create_impala_client(cls, host_port=IMPALAD):
+ def create_impala_client(cls, host_port=None, protocol='beeswax'):
+ if host_port is None:
+ host_port = cls.__get_default_host_port(protocol)
client = create_connection(host_port=host_port,
- use_kerberos=pytest.config.option.use_kerberos)
+ use_kerberos=pytest.config.option.use_kerberos, protocol=protocol)
client.connect()
return client
@classmethod
+ def __get_default_host_port(cls, protocol):
+ if protocol == 'beeswax':
+ return IMPALAD
+ else:
+ assert protocol == 'hs2'
+ return IMPALAD_HS2_HOST_PORT
+
+ @classmethod
+ def __get_cluster_host_ports(cls, protocol):
+ """Return a list of host/port combinations for all impalads in the cluster."""
+ if protocol == 'beeswax':
+ return IMPALAD_HOST_PORT_LIST
+ else:
+ assert protocol == 'hs2'
+ # TODO: support running tests against multiple coordinators for HS2. It should work,
+ # we just need to update all test runners to pass in all host/port combinations for
+ # the cluster and then handle it here.
+ raise NotImplementedError(
+ "Not yet implemented: only one HS2 host/port can be configured")
+
+ @classmethod
def create_impala_service(cls, host_port=IMPALAD, webserver_port=25000):
host, port = host_port.split(':')
return ImpaladService(host, beeswax_port=port, webserver_port=webserver_port)
@@ -208,13 +242,9 @@ class ImpalaTestSuite(BaseTestSuite):
"""
# Populate the default query option if it's empty.
if not self.default_query_options:
- try:
- query_options = impalad_client.get_default_configuration()
- for query_option in query_options:
- self.default_query_options[query_option.key.upper()] = query_option.value
- except Exception as e:
- LOG.info('Unexpected exception when getting default query options: ' + str(e))
- return
+ query_options = impalad_client.get_default_configuration()
+ for key, value in query_options.iteritems():
+ self.default_query_options[key.upper()] = value
# Restore all the changed query options.
for query_option in query_options_changed:
query_option = query_option.upper()
@@ -295,7 +325,7 @@ class ImpalaTestSuite(BaseTestSuite):
actual values are easily compared.
"""
replace_filenames_with_placeholder = True
- for section_name in ('RESULTS', 'ERRORS'):
+ for section_name in ('RESULTS', 'DBAPI_RESULTS', 'ERRORS'):
if section_name in test_section:
if "$NAMENODE" in test_section[section_name]:
replace_filenames_with_placeholder = False
@@ -305,8 +335,22 @@ class ImpalaTestSuite(BaseTestSuite):
.replace('$USER', getuser())
if use_db:
test_section[section_name] = test_section[section_name].replace('$DATABASE', use_db)
+ result_section, type_section = 'RESULTS', 'TYPES'
+ if vector.get_value('protocol') == 'hs2':
+ if 'DBAPI_RESULTS' in test_section:
+ assert 'RESULTS' in test_section,\
+ "Base RESULTS section must always be included alongside DBAPI_RESULTS"
+ # In some cases Impyla (the HS2 dbapi client) is expected to return different
+ # results, so use the dbapi-specific section if present.
+ result_section = 'DBAPI_RESULTS'
+ if 'HS2_TYPES' in test_section:
+ assert 'TYPES' in test_section,\
+ "Base TYPES section must always be included alongside HS2_TYPES"
+ # In some cases HS2 types are expected differ from Beeswax types (e.g. see
+ # IMPALA-914), so use the HS2-specific section if present.
+ type_section = 'HS2_TYPES'
verify_raw_results(test_section, result, vector.get_value('table_format').file_format,
- pytest.config.option.update_results,
+ result_section, type_section, pytest.config.option.update_results,
replace_filenames_with_placeholder)
@@ -318,7 +362,10 @@ class ImpalaTestSuite(BaseTestSuite):
Runs the query using targeting the file format/compression specified in the test
vector and the exec options specified in the test vector. If multiple_impalad=True
a connection to a random impalad will be chosen to execute each test section.
- Otherwise, the default impalad client will be used.
+ Otherwise, the default impalad client will be used. If 'protocol' (either 'hs2' or
+ 'beeswax') is set in the vector, a client for that protocol is used. Otherwise we
+ use the default: beeswax.
+
Additionally, the encoding for all test data can be specified using the 'encoding'
parameter. This is useful when data is ingested in a different encoding (ex.
latin). If not set, the default system encoding will be used.
@@ -328,6 +375,7 @@ class ImpalaTestSuite(BaseTestSuite):
"""
table_format_info = vector.get_value('table_format')
exec_options = vector.get_value('exec_option')
+ protocol = vector.get_value('protocol')
# Resolve the current user's primary group name.
group_id = pwd.getpwnam(getuser()).pw_gid
@@ -336,9 +384,14 @@ class ImpalaTestSuite(BaseTestSuite):
target_impalad_clients = list()
if multiple_impalad:
target_impalad_clients =\
- map(ImpalaTestSuite.create_impala_client, IMPALAD_HOST_PORT_LIST)
+ [ImpalaTestSuite.create_impala_client(host_port, protocol=protocol)
+ for host_port in self.__get_cluster_host_ports(protocol)]
else:
- target_impalad_clients = [self.client]
+ if protocol == 'beeswax':
+ target_impalad_clients = [self.client]
+ else:
+ assert protocol == 'hs2'
+ target_impalad_clients = [self.hs2_client]
# Change the database to reflect the file_format, compression codec etc, or the
# user specified database for all targeted impalad.
@@ -402,7 +455,7 @@ class ImpalaTestSuite(BaseTestSuite):
if 'USER' in test_section:
# Create a new client so the session will use the new username.
user = test_section['USER'].strip()
- target_impalad_client = self.create_impala_client()
+ target_impalad_client = self.create_impala_client(protocol=protocol)
for query in query.split(';'):
set_pattern_match = SET_PATTERN.match(query)
if set_pattern_match != None:
@@ -463,8 +516,8 @@ class ImpalaTestSuite(BaseTestSuite):
test_section['DML_RESULTS_TABLE']
dml_result = self.__execute_query(target_impalad_client, dml_results_query)
verify_raw_results(test_section, dml_result,
- vector.get_value('table_format').file_format,
- pytest.config.option.update_results, result_section='DML_RESULTS')
+ vector.get_value('table_format').file_format, result_section='DML_RESULTS',
+ update_section=pytest.config.option.update_results)
if pytest.config.option.update_results:
output_file = os.path.join(EE_TEST_LOGS_DIR,
test_file_name.replace('/','_') + ".test")
@@ -630,13 +683,6 @@ class ImpalaTestSuite(BaseTestSuite):
if query_options is not None: impalad_client.set_configuration(query_options)
return impalad_client.execute(query, user=user)
- def __execute_query_new_client(self, query, query_options=None,
- use_kerberos=False):
- """Executes the given query against the specified Impalad"""
- new_client = self.create_impala_client()
- new_client.set_configuration(query_options)
- return new_client.execute(query)
-
def __reset_table(self, db_name, table_name):
"""Resets a table (drops and recreates the table)"""
table = self.hive_client.get_table(db_name, table_name)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/test_dimensions.py
----------------------------------------------------------------------
diff --git a/tests/common/test_dimensions.py b/tests/common/test_dimensions.py
index af1f8e1..3413266 100644
--- a/tests/common/test_dimensions.py
+++ b/tests/common/test_dimensions.py
@@ -113,6 +113,26 @@ def create_avro_snappy_dimension(workload):
return ImpalaTestDimension('table_format',
TableFormatInfo.create_from_string(dataset, 'avro/snap/block'))
+
+def create_beeswax_hs2_dimension():
+ return ImpalaTestDimension('protocol', 'beeswax', 'hs2')
+
+
+def hs2_parquet_constraint(v):
+ """Constraint function, used to only run HS2 against Parquet format, because file format
+ and the client protocol are orthogonal."""
+ return (v.get_value('protocol') == 'beeswax' or
+ v.get_value('table_format').file_format == 'parquet' and
+ v.get_value('table_format').compression_codec == 'none')
+
+
+def hs2_text_constraint(v):
+ """Constraint function, used to only run HS2 against uncompressed text, because file
+ format and the client protocol are orthogonal."""
+ return (v.get_value('protocol') == 'beeswax' or
+ v.get_value('table_format').file_format == 'text' and
+ v.get_value('table_format').compression_codec == 'none')
+
# Common sets of values for the exec option vectors
ALL_BATCH_SIZES = [0]
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/common/test_result_verifier.py
----------------------------------------------------------------------
diff --git a/tests/common/test_result_verifier.py b/tests/common/test_result_verifier.py
index f071e6b..285b875 100644
--- a/tests/common/test_result_verifier.py
+++ b/tests/common/test_result_verifier.py
@@ -85,7 +85,7 @@ class ResultRow(object):
self.regex = try_compile_regex(row_string)
def __parse_row(self, row_string, column_types, column_labels):
- """Parses a row string and build a list of ResultColumn objects"""
+ """Parses a row string (from Beeswax) and build a list of ResultColumn objects"""
column_values = list()
if not row_string:
return column_values
@@ -314,8 +314,10 @@ def apply_error_match_filter(error_list, replace_filenames=True):
return re.sub(r'Backend \d+:', '', row)
return [replace_fn(row) for row in error_list]
-def verify_raw_results(test_section, exec_result, file_format, update_section=False,
- replace_filenames=True, result_section='RESULTS'):
+
+def verify_raw_results(test_section, exec_result, file_format, result_section,
+ type_section='TYPES', update_section=False,
+ replace_filenames=True):
"""
Accepts a raw exec_result object and verifies it matches the expected results,
including checking the ERRORS, TYPES, and LABELS test sections.
@@ -328,6 +330,9 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
The result_section parameter can be used to make this function check the results in
a DML_RESULTS section instead of the regular RESULTS section.
+
+ The 'type_section' parameter can be used to make this function check the types against
+ an alternative section from the default TYPES.
TODO: separate out the handling of sections like ERRORS from checking of query results
to allow regular RESULTS/ERRORS sections in tests with DML_RESULTS (IMPALA-4471).
"""
@@ -350,11 +355,11 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
else:
raise
- if 'TYPES' in test_section:
+ if type_section in test_section:
# Distinguish between an empty list and a list with an empty string.
- expected_types = list()
- if test_section.get('TYPES'):
- expected_types = [c.strip().upper() for c in test_section['TYPES'].rstrip('\n').split(',')]
+ section = test_section[type_section]
+ expected_types = [c.strip().upper()
+ for c in remove_comments(section).rstrip('\n').split(',')]
# Avro and Kudu represent TIMESTAMP columns as strings, so tests using TIMESTAMP are
# skipped because results will be wrong.
@@ -371,7 +376,7 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
LOG.info("Skipping type verification of Avro-format table.")
actual_types = expected_types
else:
- actual_types = parse_column_types(exec_result.schema)
+ actual_types = exec_result.column_types
try:
verify_results(expected_types, actual_types, order_matters=True)
@@ -386,8 +391,8 @@ def verify_raw_results(test_section, exec_result, file_format, update_section=Fa
actual_types = ['BIGINT']
actual_labels = ['DUMMY_LABEL']
- if exec_result and exec_result.schema:
- actual_labels = parse_column_labels(exec_result.schema)
+ if exec_result and exec_result.column_labels:
+ actual_labels = exec_result.column_labels
if 'LABELS' in test_section:
assert actual_labels is not None
@@ -441,22 +446,11 @@ def contains_order_by(query):
"""Returns true of the query contains an 'order by' clause"""
return re.search( r'order\s+by\b', query, re.M|re.I) is not None
-def parse_column_types(schema):
- """Enumerates all field schemas and returns a list of column type strings"""
- return [fs.type.upper() for fs in schema.fieldSchemas]
-
-def parse_column_labels(schema):
- """Enumerates all field schemas and returns a list of column label strings"""
- # This is an statement doesn't return a field schema (insert statement).
- if schema is None or schema.fieldSchemas is None: return list()
- return [fs.name.upper() for fs in schema.fieldSchemas]
-
def create_query_result(exec_result, order_matters=False):
"""Creates query result in the test format from the result returned from a query"""
- col_labels = parse_column_labels(exec_result.schema)
- col_types = parse_column_types(exec_result.schema)
data = parse_result_rows(exec_result)
- return QueryTestResult(data, col_types, col_labels, order_matters)
+ return QueryTestResult(data, exec_result.column_types, exec_result.column_labels,
+ order_matters)
def parse_result_rows(exec_result):
"""
@@ -467,17 +461,17 @@ def parse_result_rows(exec_result):
return []
# If the schema is 'None' assume this is an insert statement
- if exec_result.schema is None:
+ if exec_result.column_labels is None:
return raw_result
result = list()
- col_types = parse_column_types(exec_result.schema)
+ col_types = exec_result.column_types or []
for row in exec_result.data:
cols = row.split('\t')
assert len(cols) == len(col_types)
new_cols = list()
for i in xrange(len(cols)):
- if col_types[i] == 'STRING' or col_types[i] == 'CHAR':
+ if col_types[i] in ['STRING', 'CHAR', 'VARCHAR']:
col = cols[i].encode('unicode_escape')
new_cols.append("'%s'" % col)
else:
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/conftest.py
----------------------------------------------------------------------
diff --git a/tests/conftest.py b/tests/conftest.py
index 58f64a4..1eb5c76 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -78,9 +78,9 @@ def pytest_addoption(parser):
"format: workload:exploration_strategy. Ex: tpch:core,tpcds:pairwise.")
parser.addoption("--impalad", default=DEFAULT_IMPALADS,
- help="A comma-separated list of impalad host:ports to target. Note: "
- "Not all tests make use of all impalad, some tests target just the "
- "first item in the list (it is considered the 'default'")
+ help="A comma-separated list of impalad Beeswax host:ports to target. "
+ "Note: Not all tests make use of all impalad, some tests target just "
+ "the first item in the list (it is considered the 'default'")
parser.addoption("--impalad_hs2_port", default=DEFAULT_IMPALAD_HS2_PORT,
help="The impalad HiveServer2 port.")
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/custom_cluster/test_coordinators.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index b49e328..703f779 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -17,6 +17,7 @@
#
# The base class that should be used for almost all Impala tests
+import logging
import pytest
import os
import time
@@ -24,6 +25,9 @@ from subprocess import check_call
from tests.util.filesystem_utils import get_fs_path
from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+LOG = logging.getLogger('test_coordinators')
+LOG.setLevel(level=logging.DEBUG)
+
class TestCoordinators(CustomClusterTestSuite):
@pytest.mark.execute_serially
def test_multiple_coordinators(self):
@@ -43,14 +47,16 @@ class TestCoordinators(CustomClusterTestSuite):
beeswax_client = None
try:
beeswax_client = worker.service.create_beeswax_client()
- except: pass
+ except Exception, e:
+ LOG.info("Caught exception {0}".format(e))
finally:
assert beeswax_client is None
hs2_client = None
try:
hs2_client = worker.service.create_hs2_client()
- except: pass
+ except Exception, e:
+ LOG.info("Caught exception {0}".format(e))
finally:
assert hs2_client is None
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/custom_cluster/test_session_expiration.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_session_expiration.py b/tests/custom_cluster/test_session_expiration.py
index 78862ec..027cc41 100644
--- a/tests/custom_cluster/test_session_expiration.py
+++ b/tests/custom_cluster/test_session_expiration.py
@@ -29,12 +29,7 @@ class TestSessionExpiration(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args("--idle_session_timeout=6")
def test_session_expiration(self, vector):
impalad = self.cluster.get_any_impalad()
- # setup_class creates an Impala client to <hostname>:21000 after the cluster starts.
- # The client expires at the same time as the client created below. Since we choose the
- # impalad to connect to randomly, the test becomes flaky, as the metric we expect to
- # be incremented by 1 gets incremented by 2 if both clients are connected to the same
- # Impalad.
- self.client.close()
+ self.__close_default_clients()
num_expired = impalad.service.get_metric_value("impala-server.num-sessions-expired")
client = impalad.service.create_beeswax_client()
# Sleep for half the expiration time to confirm that the session is not expired early
@@ -50,7 +45,7 @@ class TestSessionExpiration(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args("--idle_session_timeout=3")
def test_session_expiration_with_set(self, vector):
impalad = self.cluster.get_any_impalad()
- self.client.close()
+ self.__close_default_clients()
num_expired = impalad.service.get_metric_value("impala-server.num-sessions-expired")
# Test if we can set a shorter timeout than the process-wide option
@@ -72,7 +67,7 @@ class TestSessionExpiration(CustomClusterTestSuite):
@CustomClusterTestSuite.with_args("--idle_session_timeout=5")
def test_unsetting_session_expiration(self, vector):
impalad = self.cluster.get_any_impalad()
- self.client.close()
+ self.__close_default_clients()
num_expired = impalad.service.get_metric_value("impala-server.num-sessions-expired")
# Test unsetting IDLE_SESSION_TIMEOUT
@@ -109,3 +104,10 @@ class TestSessionExpiration(CustomClusterTestSuite):
queued_query_profile = impalad.service.create_beeswax_client().get_runtime_profile(
queued_handle)
assert "Admission result: Cancelled (queued)" in queued_query_profile
+
+ def __close_default_clients(self):
+ """Close the clients that were automatically created by setup_class(). These clients
+ can expire during test, which results in metrics that tests depend on changing. Each
+ test should create its own clients as needed."""
+ self.client.close()
+ self.hs2_client.close()
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/metadata/test_views_compatibility.py
----------------------------------------------------------------------
diff --git a/tests/metadata/test_views_compatibility.py b/tests/metadata/test_views_compatibility.py
index f4987c3..ac208d7 100644
--- a/tests/metadata/test_views_compatibility.py
+++ b/tests/metadata/test_views_compatibility.py
@@ -103,7 +103,7 @@ class TestViewCompatibility(ImpalaTestSuite):
# The table may or may not have been created in Hive. And so, "invalidate metadata"
# may throw an exception.
try:
- self.client.invalidate_table(test_case.hive_view_name)
+ self.client.execute("invalidate metadata {0}".format(test_case.hive_view_name))
except ImpalaBeeswaxException as e:
assert "TableNotFoundException" in str(e)
@@ -135,7 +135,7 @@ class TestViewCompatibility(ImpalaTestSuite):
self._exec_in_hive(test_case.get_drop_view_sql('HIVE'),\
test_case.get_create_view_sql('HIVE'), None)
try:
- self.client.invalidate_table(test_case.hive_view_name)
+ self.client.execute("invalidate metadata {0}".format(test_case.hive_view_name))
except ImpalaBeeswaxException as e:
assert "TableNotFoundException" in str(e)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_aggregation.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_aggregation.py b/tests/query_test/test_aggregation.py
index 55233fa..c62f729 100644
--- a/tests/query_test/test_aggregation.py
+++ b/tests/query_test/test_aggregation.py
@@ -27,8 +27,6 @@ from tests.common.test_dimensions import (
create_uncompressed_text_dimension)
from tests.common.test_result_verifier import (
assert_codegen_enabled,
- parse_column_types,
- parse_column_labels,
QueryTestResult,
parse_result_rows)
from tests.common.test_vector import ImpalaTestDimension
@@ -397,8 +395,8 @@ class TestWideAggregationQueries(ImpalaTestSuite):
# All rows should be distinct.
expected_result = widetable.get_data(1000, 10, quote_strings=True)
- types = parse_column_types(result.schema)
- labels = parse_column_labels(result.schema)
+ types = result.column_types
+ labels = result.column_labels
expected = QueryTestResult(expected_result, types, labels, order_matters=False)
actual = QueryTestResult(parse_result_rows(result), types, labels,
order_matters=False)
[7/8] impala git commit: IMPALA-7654: [DOCS] TRUNCATE does not put
HDFS data files to the trash
Posted by bi...@apache.org.
IMPALA-7654: [DOCS] TRUNCATE does not put HDFS data files to the trash
Change-Id: I4930b2345515abf9b086575a476d20c87117e5c4
Reviewed-on: http://gerrit.cloudera.org:8080/11583
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/7b131172
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/7b131172
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/7b131172
Branch: refs/heads/master
Commit: 7b13117293380821c28df81ed8c308623e835dd5
Parents: ee3da43
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Thu Oct 4 13:51:43 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Tue Oct 9 19:18:10 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_truncate_table.xml | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/7b131172/docs/topics/impala_truncate_table.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_truncate_table.xml b/docs/topics/impala_truncate_table.xml
index e40802c..fc9affd 100644
--- a/docs/topics/impala_truncate_table.xml
+++ b/docs/topics/impala_truncate_table.xml
@@ -66,9 +66,6 @@ under the License.
effect. This capability is useful in standardized setup scripts that are
might be run both before and after some of the tables exist. This clause
is available in <keyword keyref="impala25_full"/> and higher. </p>
- <p> Any HDFS data files removed by this statement go into the HDFS trashcan,
- from which you can recover them within a defined time interval if this
- operation turns out to be a mistake. </p>
<p conref="../shared/impala_common.xml#common/disk_space_blurb"/>
<p conref="../shared/impala_common.xml#common/s3_blurb"/>
<p rev="2.2.0"> Although Impala cannot write new data to a table stored in
[6/8] impala git commit: Prettify the timeline produced by
test-with-docker.py
Posted by bi...@apache.org.
Prettify the timeline produced by test-with-docker.py
The change tweaks the HTML template for the timeline summary
to make it slightly more readable:
- Adds legend strings to the CPU graphs
- Inserts the test run name into the CPU chart title to clarify
which chart show which build/test phase
- Stretches the CPU charts a bit wider
- Identifes the common prefix of the phase/container names (the build
name) and delete it from the chart labels. This increases legibility
by cutting down on noise and growing the chart real estate.
To support this change the Python drivers are also changed:
the build name parameter, which is the common prefix, is passed
to monitor.py and written to the JSON output
- The name of the build and data load phase container is suffixed with
"-build" so that it shares the naming convention for the other
containers.
- The timeline graph section is sized explicitly byt computing the height
from the number of distinct tasks. This avoids having a second scrollbar
for the timeline, which is annoying.
The formula is pretty crude: it uses empirical constants, but produces
an OK layout for the default font sizes in Chrome (both on Linux
and the Mac).
Tested so far by tweaking the HTML template and an HTML result file
from an earlier build.
Change-Id: I7a41bea762b0e33f3d71b0be57eedbacb19c680c
Reviewed-on: http://gerrit.cloudera.org:8080/11578
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/ee3da437
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/ee3da437
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/ee3da437
Branch: refs/heads/master
Commit: ee3da43709895450bded45a6aa9bccca947c6add
Parents: 122c366
Author: Laszlo Gaal <la...@cloudera.com>
Authored: Wed Sep 26 15:17:27 2018 +0200
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 9 19:12:50 2018 +0000
----------------------------------------------------------------------
docker/monitor.py | 7 +++---
docker/test-with-docker.py | 5 ++--
docker/timeline.html.template | 47 +++++++++++++++++++++++++++++++-------
3 files changed, 46 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/ee3da437/docker/monitor.py
----------------------------------------------------------------------
diff --git a/docker/monitor.py b/docker/monitor.py
index 58e2a83..5eefe5c 100644
--- a/docker/monitor.py
+++ b/docker/monitor.py
@@ -217,10 +217,11 @@ class Timeline(object):
the charts, which happens in the browser.
"""
- def __init__(self, monitor_file, containers, interesting_re):
+ def __init__(self, monitor_file, containers, interesting_re, buildname):
self.monitor_file = monitor_file
self.containers = containers
self.interesting_re = interesting_re
+ self.buildname = buildname
def logfile_timeline(self, container):
"""Returns a list of (name, timestamp, line) tuples for interesting lines in
@@ -340,7 +341,7 @@ class Timeline(object):
template_path = os.path.join(os.path.dirname(__file__), "timeline.html.template")
shutil.copyfileobj(file(template_path), o)
o.write("\n<script>\nvar data = \n")
- json.dump(dict(timeline=timeline_json, metrics=metrics_by_container,
- max_ts=(max_metric_ts - min_ts)), o, indent=2)
+ json.dump(dict(buildname=self.buildname, timeline=timeline_json,
+ metrics=metrics_by_container, max_ts=(max_metric_ts - min_ts)), o, indent=2)
o.write("</script>")
o.close()
http://git-wip-us.apache.org/repos/asf/impala/blob/ee3da437/docker/test-with-docker.py
----------------------------------------------------------------------
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index dbeef81..4efd712 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -616,7 +616,7 @@ class TestWithDocker(object):
def _create_build_image(self):
"""Creates the "build image", with Impala compiled and data loaded."""
container = self._create_container(
- image="ubuntu:16.04", name=self.name,
+ image="ubuntu:16.04", name=self.name + "-build",
logdir="build",
logname="log-build.txt",
# entrypoint.sh will create a user with our uid; this
@@ -709,7 +709,8 @@ class TestWithDocker(object):
timeline = monitor.Timeline(
monitor_file=self.monitoring_output_file,
containers=self.containers,
- interesting_re=self._INTERESTING_RE)
+ interesting_re=self._INTERESTING_RE,
+ buildname=self.name)
timeline.create(os.path.join(self.log_dir, "timeline.html"))
def log_summary(self):
http://git-wip-us.apache.org/repos/asf/impala/blob/ee3da437/docker/timeline.html.template
----------------------------------------------------------------------
diff --git a/docker/timeline.html.template b/docker/timeline.html.template
index d2960d7..2add9a6 100644
--- a/docker/timeline.html.template
+++ b/docker/timeline.html.template
@@ -104,32 +104,61 @@ function drawChart() {
// timeofday isn't supported here
dataTable.addColumn({ type: 'datetime', id: 'Start' });
dataTable.addColumn({ type: 'datetime', id: 'End' });
+
+ // Extract the build name, which is a common prefix of the container names.
+ // Add one to the length we know that the driver script adds a hyphen when building
+ //the container names.
+ var buildnamePrefix = data.buildname;
+ var commonPrefix = buildnamePrefix.length + 1;
// Timeline
+ const rowLabels = new Set();
for (i = 0; i < data.timeline.length; ++i) {
var row = data.timeline[i];
- dataTable.addRow([ row[0], row[1], ts_to_date(row[2]), ts_to_date(row[3]) ]);
+ // Hack up the label in row[0]: cut off the common prefix, which is the build
+ // name. This is the same for all shards, so it is not useful for the charts.
+ var rowLabel = row[0].slice(commonPrefix);
+ if (rowLabel == "") rowLabel = ' -- ';
+ dataTable.addRow([ rowLabel, row[1], ts_to_date(row[2]), ts_to_date(row[3]) ]);
+ rowLabels.add(rowLabel);
}
- chart.draw(dataTable, { height: "400px" } );
+ // Precompute timeline height to get rid of the timeline's own scrollbar.
+ const laneHeight = 41;
+ const axisHeight = 50;
+
+ timelineHeight = rowLabels.size * laneHeight + axisHeight;
+ var options = {
+ // Width is specified in the <div> tag below; for some reason it doesn't have the
+ // same effect when specified here.
+ // Height is precomputed from the incoming data, so we set it here.
+ height: timelineHeight,
+ timeline: { showRowLabels : 'true' }
+ };
+ chart.draw(dataTable, options);
for (const k of Object.keys(data.metrics)) {
var lineChart = document.createElement("div");
lineChartContainer.appendChild(lineChart);
var dataTable = new google.visualization.DataTable();
- dataTable.addColumn({ type: 'timeofday', id: 'Time' });
- dataTable.addColumn({ type: 'number', id: 'User' });
- dataTable.addColumn({ type: 'number', id: 'System' });
+ dataTable.addColumn({ type: 'timeofday', label: 'Time', id: 'Time' });
+ dataTable.addColumn({ type: 'number', label: 'User', id: 'User' });
+ dataTable.addColumn({ type: 'number', label: 'System', id: 'System' });
for (const row of data.metrics[k]) {
dataTable.addRow([ ts_to_hms(row[0]), row[1], row[2] ]);
}
+ // De-clutter the timeline labels: Slice off the common prefix of the build name
+ var title = k.slice(commonPrefix);
+ if (title == "") title = ' -- ';
var options = {
- title: 'CPU',
+ title: 'CPU: ' + title,
legend: { position: 'bottom' },
hAxis: {
minValue: [0, 0, 0],
maxValue: ts_to_hms(data.max_ts)
- }
+ },
+ // Stretch the CPU graph a bit wider than the default
+ chartArea: {width: '95%'},
};
var chart = new google.visualization.LineChart(lineChart);
@@ -137,5 +166,7 @@ function drawChart() {
}
}
</script>
-<div id="timelineContainer" style="height: 400px;"></div>
+<!-- Set width here, but leave height unset,
+ it is computed above in the JavaScript code. -->
+<div id="timelineContainer" style="width: 99%;"></div>
<div id="lineChartContainer" style="height: 200px;"></div>
[5/8] impala git commit: IMPALA-7675: Fix the error handling of
UpdateTableUsage() RPC
Posted by bi...@apache.org.
IMPALA-7675: Fix the error handling of UpdateTableUsage() RPC
UpdateTableUsage() is logically a one-way RPC and the status object in
TUpdateTableUsageResponse is set only if there is an error at RPC
layer. This patch fixes the incorrect error handling that leads to
NullPointerException in ImpaladTableUsageTracer.
Change-Id: Iccba4c6f4696ef08bc8a614ae13f62b5e445917b
Reviewed-on: http://gerrit.cloudera.org:8080/11603
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/122c3667
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/122c3667
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/122c3667
Branch: refs/heads/master
Commit: 122c3667dac4a604adbe6dbe90030f8801b579bc
Parents: d05f73f
Author: Tianyi Wang <tw...@cloudera.com>
Authored: Fri Oct 5 18:24:09 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Tue Oct 9 06:09:24 2018 +0000
----------------------------------------------------------------------
be/src/service/fe-support.cc | 4 ++--
.../java/org/apache/impala/catalog/ImpaladTableUsageTracker.java | 3 +--
2 files changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/122c3667/be/src/service/fe-support.cc
----------------------------------------------------------------------
diff --git a/be/src/service/fe-support.cc b/be/src/service/fe-support.cc
index b933c7d..5dd06de 100644
--- a/be/src/service/fe-support.cc
+++ b/be/src/service/fe-support.cc
@@ -540,7 +540,7 @@ Java_org_apache_impala_service_FeSupport_NativeUpdateTableUsage(
if (!status.ok()) {
LOG(ERROR) << status.GetDetail();
status.AddDetail("Error making an RPC call to Catalog server.");
- status.ToThrift(&result.status);
+ status.SetTStatus(&result);
}
jbyteArray result_bytes = nullptr;
@@ -583,7 +583,7 @@ Java_org_apache_impala_service_FeSupport_NativeGetPartitionStats(
Status status = catalog_op_executor.GetPartitionStats(request, &result);
if (!status.ok()) {
LOG(ERROR) << status.GetDetail();
- status.ToThrift(&result.status);
+ status.SetTStatus(&result);
}
jbyteArray result_bytes = nullptr;
THROW_IF_ERROR_RET(SerializeThriftMsg(env, &result, &result_bytes), env,
http://git-wip-us.apache.org/repos/asf/impala/blob/122c3667/fe/src/main/java/org/apache/impala/catalog/ImpaladTableUsageTracker.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ImpaladTableUsageTracker.java b/fe/src/main/java/org/apache/impala/catalog/ImpaladTableUsageTracker.java
index 45ba1ab..2d6ccda 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ImpaladTableUsageTracker.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ImpaladTableUsageTracker.java
@@ -107,8 +107,7 @@ public class ImpaladTableUsageTracker {
FeSupport.NativeUpdateTableUsage(new TSerializer().serialize(reqToSend));
TUpdateTableUsageResponse resp = new TUpdateTableUsageResponse();
JniUtil.deserializeThrift(new TBinaryProtocol.Factory(), resp, byteResp);
- if (resp.status.isSetStatus_code() &&
- !resp.status.status_code.equals(TErrorCode.OK)) {
+ if (resp.isSetStatus() && !resp.status.status_code.equals(TErrorCode.OK)) {
LOG.warn(
updateFailureMessage + Joiner.on("\n").join(resp.status.getError_msgs()));
}
[3/8] impala git commit: IMPALA-7647: Add HS2/Impyla dimension to
TestQueries
Posted by bi...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_chars.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_chars.py b/tests/query_test/test_chars.py
index 86ec095..939e07e 100644
--- a/tests/query_test/test_chars.py
+++ b/tests/query_test/test_chars.py
@@ -18,7 +18,8 @@
import pytest
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.test_dimensions import create_exec_option_dimension
+from tests.common.test_dimensions import (create_exec_option_dimension,
+ create_beeswax_hs2_dimension, hs2_parquet_constraint, hs2_text_constraint)
from tests.util.filesystem_utils import get_fs_path
class TestStringQueries(ImpalaTestSuite):
@@ -34,11 +35,17 @@ class TestStringQueries(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format in ['text'] and
v.get_value('table_format').compression_codec in ['none'])
+ # Run these queries through both beeswax and HS2 to get coverage of CHAR/VARCHAR
+ # returned via both protocols.
+ cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension())
+ cls.ImpalaTestMatrix.add_constraint(hs2_text_constraint)
def test_chars(self, vector):
self.run_test_case('QueryTest/chars', vector)
def test_chars_tmp_tables(self, vector, unique_database):
+ if vector.get_value('protocol') == 'hs2':
+ pytest.skip("HS2 does not return row counts for inserts")
# Tests that create temporary tables and require a unique database.
self.run_test_case('QueryTest/chars-tmp-tables', vector, unique_database)
@@ -59,6 +66,10 @@ class TestCharFormats(ImpalaTestSuite):
v.get_value('table_format').file_format in ['orc'] or
(v.get_value('table_format').file_format in ['text'] and
v.get_value('table_format').compression_codec in ['none']))
+ # Run these queries through both beeswax and HS2 to get coverage of CHAR/VARCHAR
+ # returned via both protocols.
+ cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension())
+ cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
def test_char_format(self, vector):
self.run_test_case('QueryTest/chars-formats', vector)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_decimal_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_decimal_queries.py b/tests/query_test/test_decimal_queries.py
index 45a702d..a156321 100644
--- a/tests/query_test/test_decimal_queries.py
+++ b/tests/query_test/test_decimal_queries.py
@@ -20,7 +20,8 @@
from copy import copy
from tests.common.impala_test_suite import ImpalaTestSuite
-from tests.common.test_dimensions import create_exec_option_dimension_from_dict
+from tests.common.test_dimensions import (create_exec_option_dimension_from_dict,
+ create_beeswax_hs2_dimension, hs2_parquet_constraint)
from tests.common.test_vector import ImpalaTestDimension
class TestDecimalQueries(ImpalaTestSuite):
@@ -45,6 +46,11 @@ class TestDecimalQueries(ImpalaTestSuite):
v.get_value('table_format').compression_codec == 'none') or
v.get_value('table_format').file_format in ['parquet', 'orc', 'kudu'])
+ # Run these queries through both beeswax and HS2 to get coverage of decimals returned
+ # via both protocols.
+ cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension())
+ cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
+
def test_queries(self, vector):
self.run_test_case('QueryTest/decimal', vector)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_insert.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_insert.py b/tests/query_test/test_insert.py
index 20fee41..46d5249 100644
--- a/tests/query_test/test_insert.py
+++ b/tests/query_test/test_insert.py
@@ -27,8 +27,6 @@ from tests.common.test_dimensions import (
create_exec_option_dimension,
create_uncompressed_text_dimension)
from tests.common.test_result_verifier import (
- parse_column_types,
- parse_column_labels,
QueryTestResult,
parse_result_rows)
from tests.common.test_vector import ImpalaTestDimension
@@ -201,8 +199,8 @@ class TestInsertWideTable(ImpalaTestSuite):
assert result.data == ["1"]
result = self.client.execute("select * from " + table_name)
- types = parse_column_types(result.schema)
- labels = parse_column_labels(result.schema)
+ types = result.column_types
+ labels = result.column_labels
expected = QueryTestResult([col_vals], types, labels, order_matters=False)
actual = QueryTestResult(parse_result_rows(result), types, labels, order_matters=False)
assert expected == actual
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_queries.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_queries.py b/tests/query_test/test_queries.py
index 90cb392..16a6d9f 100644
--- a/tests/query_test/test_queries.py
+++ b/tests/query_test/test_queries.py
@@ -23,7 +23,9 @@ import re
from tests.common.impala_test_suite import ImpalaTestSuite
from tests.common.skip import SkipIfEC
-from tests.common.test_dimensions import create_uncompressed_text_dimension, extend_exec_option_dimension
+from tests.common.test_dimensions import (
+ create_uncompressed_text_dimension, extend_exec_option_dimension,
+ create_beeswax_hs2_dimension, hs2_parquet_constraint)
from tests.common.test_vector import ImpalaTestVector
class TestQueries(ImpalaTestSuite):
@@ -33,6 +35,11 @@ class TestQueries(ImpalaTestSuite):
if cls.exploration_strategy() == 'core':
cls.ImpalaTestMatrix.add_constraint(lambda v:\
v.get_value('table_format').file_format == 'parquet')
+ # Run these queries through both beeswax and HS2 to get coverage of both protocols.
+ # Don't run all combinations of table format and protocol - the dimensions should
+ # be orthogonal.
+ cls.ImpalaTestMatrix.add_dimension(create_beeswax_hs2_dimension())
+ cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
# Adding a test dimension here to test the small query opt in exhaustive.
if cls.exploration_strategy() == 'exhaustive':
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index ab724d2..0000867 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -48,8 +48,6 @@ from tests.common.file_utils import (
create_table_from_parquet,
create_table_and_copy_files)
from tests.common.test_result_verifier import (
- parse_column_types,
- parse_column_labels,
QueryTestResult,
parse_result_rows)
from tests.common.test_vector import ImpalaTestDimension
@@ -275,8 +273,8 @@ class TestWideTable(ImpalaTestSuite):
assert len(result.data) == NUM_ROWS
return
- types = parse_column_types(result.schema)
- labels = parse_column_labels(result.schema)
+ types = result.column_types
+ labels = result.column_labels
expected = QueryTestResult(expected_result, types, labels, order_matters=False)
actual = QueryTestResult(parse_result_rows(result), types, labels,
order_matters=False)
http://git-wip-us.apache.org/repos/asf/impala/blob/d05f73f4/tests/util/test_file_parser.py
----------------------------------------------------------------------
diff --git a/tests/util/test_file_parser.py b/tests/util/test_file_parser.py
index 8d3c58a..ccd0af5 100644
--- a/tests/util/test_file_parser.py
+++ b/tests/util/test_file_parser.py
@@ -95,7 +95,7 @@ def parse_query_test_file(file_name, valid_section_names=None, encoding=None):
section_names = valid_section_names
if section_names is None:
section_names = ['QUERY', 'RESULTS', 'TYPES', 'LABELS', 'SETUP', 'CATCH', 'ERRORS',
- 'USER', 'RUNTIME_PROFILE', 'SHELL', 'DML_RESULTS']
+ 'USER', 'RUNTIME_PROFILE', 'SHELL', 'DML_RESULTS', 'DBAPI_RESULTS', 'HS2_TYPES']
return parse_test_file(file_name, section_names, encoding=encoding,
skip_unknown_sections=False)