You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ab...@apache.org on 2016/09/14 00:47:59 UTC
[3/3] incubator-impala git commit: IMPALA-4122: qgen: fix bitrotted
cluster unit tests
IMPALA-4122: qgen: fix bitrotted cluster unit tests
There's a small set of pytest-style tests and associated conftest for
testing some of the cluster-related test infrastructure Python objects.
Going forward, I want unit tests for the query generator to be run as
part of patch acceptance (CI isn't necessary at this time).
This patch fixes a bitrotted test and moves the tests into the
tests-for-qgen directory. The moves were performed thusly:
$ git mv conftest.py tests/
$ git mv cluster_tests.py tests/test_cluster.py
Change-Id: I3e855e265ae245ebe3691d077284ac5761909e00
Reviewed-on: http://gerrit.cloudera.org:8080/4404
Reviewed-by: Michael Brown <mi...@cloudera.com>
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Internal Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/0c874189
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/0c874189
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/0c874189
Branch: refs/heads/master
Commit: 0c874189ed902d566b3df37959c937ff6a7bfee8
Parents: d379875
Author: Michael Brown <mi...@cloudera.com>
Authored: Tue Sep 13 12:30:26 2016 -0700
Committer: Internal Jenkins <cl...@gerrit.cloudera.org>
Committed: Wed Sep 14 00:24:06 2016 +0000
----------------------------------------------------------------------
tests/comparison/cluster_tests.py | 94 -----------------------------
tests/comparison/conftest.py | 46 --------------
tests/comparison/tests/conftest.py | 62 +++++++++++++++++++
tests/comparison/tests/test_cluster.py | 94 +++++++++++++++++++++++++++++
4 files changed, 156 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/cluster_tests.py
----------------------------------------------------------------------
diff --git a/tests/comparison/cluster_tests.py b/tests/comparison/cluster_tests.py
deleted file mode 100644
index 976f423..0000000
--- a/tests/comparison/cluster_tests.py
+++ /dev/null
@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# These are unit tests for cluster.py.
-
-from time import time
-
-from tests.common.errors import Timeout
-
-from common import Column, Table
-from db_types import BigInt, String
-
-class TestCluster(object):
-
- def test_cmd(self, cluster):
- host = cluster.impala.impalads[0].host_name
- assert cluster.shell("echo -n HI", host) == "HI"
-
- try:
- cluster.shell("bad", host)
- assert False
- except Exception as e:
- assert "command not found" in str(e)
-
- start = time()
- try:
- cluster.shell("echo HI; sleep 60", host, timeout_secs=3)
- assert False
- except Timeout as e:
- assert "HI" in str(e)
- assert 3 <= time() - start <= 6
-
-
-class TestHdfs(object):
-
- def test_ls(self, cluster):
- ls = cluster.hdfs.create_client().list("/")
- assert "tmp" in ls
- assert "etc" not in ls
-
-
-class TestHive(object):
-
- def test_list_databases(self, hive_cursor):
- assert "default" in hive_cursor.list_db_names()
-
- def test_non_mr_exec(self, hive_cursor):
- hive_cursor.execute("SELECT 1")
- rows = hive_cursor.fetchall()
- assert rows
- assert rows[0][0] == 1
-
-
-class TestImpala(object):
-
- def test_list_databases(self, cursor):
- assert "default" in cursor.list_db_names()
-
- def test_exec(self, cursor):
- cursor.execute("SELECT 1")
- rows = cursor.fetchall()
- assert rows
- assert rows[0][0] == 1
-
-
-class TestModel(object):
-
- def test_table_model(self, cursor, hive_cursor):
- table = Table("some_test_table")
- cursor.drop_table(table.name, if_exists=True)
- table.storage_format = 'textfile'
- table.cols.append(Column(table, "bigint_col", BigInt))
- table.cols.append(Column(table, "string_col", String))
- cursor.create_table(table)
- try:
- other = hive_cursor.describe_table(table.name)
- assert other.name == table.name
- assert other.cols == table.cols
- finally:
- cursor.drop_table(table.name)
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/conftest.py
----------------------------------------------------------------------
diff --git a/tests/comparison/conftest.py b/tests/comparison/conftest.py
deleted file mode 100644
index 8126a84..0000000
--- a/tests/comparison/conftest.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import pytest
-
-import cli_options
-from cluster import CmCluster, MiniCluster
-
-"""This module provides pytest 'fixtures'. See cluster_tests.py for usage."""
-
-__cluster = None
-
-def pytest_addoption(parser):
- if not hasattr(parser, "add_argument"):
- parser.add_argument = parser.addoption
- cli_options.add_cm_options(parser)
-
-
-@pytest.fixture
-def cluster(request):
- global __cluster
- if not __cluster:
- cm_host = get_option_value(request, "cm_host")
- if cm_host:
- __cluster = CmCluster(cm_host, port=get_option_value(request, "cm_port"),
- user=get_option_value(request, "cm_user"),
- password=get_option_value(request, "cm_password"),
- cluster_name=get_option_value(request, "cm_cluster_name"))
- else:
- __cluster = MiniCluster()
- return __cluster
-
-
-@pytest.yield_fixture
-def hive_cursor(request):
- with cluster(request).hive.connect() as conn:
- with conn.cursor() as cur:
- yield cur
-
-
-@pytest.yield_fixture
-def cursor(request):
- with cluster(request).impala.connect() as conn:
- with conn.cursor() as cur:
- yield cur
-
-
-def get_option_value(request, dest_var_name):
- return request.config.getoption("--" + dest_var_name.replace("_", "-"))
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/tests/conftest.py
----------------------------------------------------------------------
diff --git a/tests/comparison/tests/conftest.py b/tests/comparison/tests/conftest.py
new file mode 100644
index 0000000..dd39fde
--- /dev/null
+++ b/tests/comparison/tests/conftest.py
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+
+from tests.comparison import cli_options
+from tests.comparison.cluster import CmCluster, MiniCluster
+
+
+__cluster = None
+
+def pytest_addoption(parser):
+ if not hasattr(parser, "add_argument"):
+ parser.add_argument = parser.addoption
+ cli_options.add_cm_options(parser)
+
+
+@pytest.fixture
+def cluster(request):
+ global __cluster
+ if not __cluster:
+ cm_host = get_option_value(request, "cm_host")
+ if cm_host:
+ __cluster = CmCluster(cm_host, port=get_option_value(request, "cm_port"),
+ user=get_option_value(request, "cm_user"),
+ password=get_option_value(request, "cm_password"),
+ cluster_name=get_option_value(request, "cm_cluster_name"))
+ else:
+ __cluster = MiniCluster()
+ return __cluster
+
+
+@pytest.yield_fixture
+def hive_cursor(request):
+ with cluster(request).hive.connect() as conn:
+ with conn.cursor() as cur:
+ yield cur
+
+
+@pytest.yield_fixture
+def cursor(request):
+ with cluster(request).impala.connect() as conn:
+ with conn.cursor() as cur:
+ yield cur
+
+
+def get_option_value(request, dest_var_name):
+ return request.config.getoption("--" + dest_var_name.replace("_", "-"))
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/0c874189/tests/comparison/tests/test_cluster.py
----------------------------------------------------------------------
diff --git a/tests/comparison/tests/test_cluster.py b/tests/comparison/tests/test_cluster.py
new file mode 100644
index 0000000..2e3bc54
--- /dev/null
+++ b/tests/comparison/tests/test_cluster.py
@@ -0,0 +1,94 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# These are unit tests for cluster.py.
+
+from time import time
+
+from tests.common.errors import Timeout
+
+from tests.comparison.common import Column, Table
+from tests.comparison.db_types import BigInt, String
+
+class TestCluster(object):
+
+ def test_cmd(self, cluster):
+ host = cluster.impala.impalads[0].host_name
+ assert cluster.shell("echo -n HI", host) == "HI"
+
+ try:
+ cluster.shell("bad", host)
+ assert False
+ except Exception as e:
+ assert "command not found" in str(e)
+
+ start = time()
+ try:
+ cluster.shell("echo HI; sleep 60", host, timeout_secs=3)
+ assert False
+ except Timeout as e:
+ assert "HI" in str(e)
+ assert 3 <= time() - start <= 6
+
+
+class TestHdfs(object):
+
+ def test_ls(self, cluster):
+ ls = cluster.hdfs.create_client().list("/")
+ assert "tmp" in ls
+ assert "etc" not in ls
+
+
+class TestHive(object):
+
+ def test_list_databases(self, hive_cursor):
+ assert "default" in hive_cursor.list_db_names()
+
+ def test_non_mr_exec(self, hive_cursor):
+ hive_cursor.execute("SELECT 1")
+ rows = hive_cursor.fetchall()
+ assert rows
+ assert rows[0][0] == 1
+
+
+class TestImpala(object):
+
+ def test_list_databases(self, cursor):
+ assert "default" in cursor.list_db_names()
+
+ def test_exec(self, cursor):
+ cursor.execute("SELECT 1")
+ rows = cursor.fetchall()
+ assert rows
+ assert rows[0][0] == 1
+
+
+class TestModel(object):
+
+ def test_table_model(self, cursor, hive_cursor):
+ table = Table("some_test_table")
+ cursor.drop_table(table.name, if_exists=True)
+ table.storage_format = 'textfile'
+ table.add_col(Column(table, "bigint_col", BigInt))
+ table.add_col(Column(table, "string_col", String))
+ cursor.create_table(table)
+ try:
+ other = hive_cursor.describe_table(table.name)
+ assert other.name == table.name
+ assert other.cols == table.cols
+ finally:
+ cursor.drop_table(table.name)