You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airflow.apache.org by ka...@apache.org on 2018/06/19 09:51:07 UTC
incubator-airflow git commit: [AIRFLOW-2612][AIRFLOW-2534] Clean up
Hive-related tests
Repository: incubator-airflow
Updated Branches:
refs/heads/master 5676ec79d -> ecf68c7d3
[AIRFLOW-2612][AIRFLOW-2534] Clean up Hive-related tests
There is several HiveServer2-related tests
which are not run by default in
tests/operators/hive_operator.py,
but AIRFLOW-2534 implemented almost all of them
in airflow/hooks/hive_hooks.py.
This PR removes duplicated tests from
tests/operators/hive_operator.py and
moves an unimplemented test to
airflow/hooks/hive_hooks.py with a bit of fix.
Closes #3498 from sekikn/AIRFLOW-2612
Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/ecf68c7d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/ecf68c7d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/ecf68c7d
Branch: refs/heads/master
Commit: ecf68c7d3133b2cb9847b2f21b674482cd076358
Parents: 5676ec7
Author: Kengo Seki <se...@apache.org>
Authored: Tue Jun 19 10:51:01 2018 +0100
Committer: Kaxil Naik <ka...@apache.org>
Committed: Tue Jun 19 10:51:01 2018 +0100
----------------------------------------------------------------------
tests/hooks/test_hive_hook.py | 14 ++++-
tests/operators/hive_operator.py | 113 ----------------------------------
2 files changed, 12 insertions(+), 115 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ecf68c7d/tests/hooks/test_hive_hook.py
----------------------------------------------------------------------
diff --git a/tests/hooks/test_hive_hook.py b/tests/hooks/test_hive_hook.py
index 7d5d103..81270c6 100644
--- a/tests/hooks/test_hive_hook.py
+++ b/tests/hooks/test_hive_hook.py
@@ -375,8 +375,8 @@ class TestHiveServer2Hook(unittest.TestCase):
def test_get_records(self):
hook = HiveServer2Hook()
query = "SELECT * FROM {}".format(self.table)
- results = hook.get_pandas_df(query, schema=self.database)
- self.assertEqual(len(results), 2)
+ results = hook.get_records(query, schema=self.database)
+ self.assertListEqual(results, [(1, 1), (2, 2)])
def test_get_pandas_df(self):
hook = HiveServer2Hook()
@@ -409,3 +409,13 @@ class TestHiveServer2Hook(unittest.TestCase):
self.assertListEqual(df.columns.tolist(), self.columns)
self.assertListEqual(df[self.columns[0]].values.tolist(), [1, 2])
self.assertEqual(len(df), 2)
+
+ def test_multi_statements(self):
+ sqls = [
+ "CREATE TABLE IF NOT EXISTS test_multi_statements (i INT)",
+ "SELECT * FROM {}".format(self.table),
+ "DROP TABLE test_multi_statements",
+ ]
+ hook = HiveServer2Hook()
+ results = hook.get_records(sqls, schema=self.database)
+ self.assertListEqual(results, [(1, 1), (2, 2)])
http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/ecf68c7d/tests/operators/hive_operator.py
----------------------------------------------------------------------
diff --git a/tests/operators/hive_operator.py b/tests/operators/hive_operator.py
index 0914ba9..1569e7f 100644
--- a/tests/operators/hive_operator.py
+++ b/tests/operators/hive_operator.py
@@ -114,119 +114,6 @@ if 'AIRFLOW_RUNALL_TESTS' in os.environ:
import airflow.hooks.hive_hooks
import airflow.operators.presto_to_mysql
- class HiveServer2Test(unittest.TestCase):
- def setUp(self):
- configuration.load_test_config()
- self.nondefault_schema = "nondefault"
-
- def test_select_conn(self):
- from airflow.hooks.hive_hooks import HiveServer2Hook
- sql = "select 1"
- hook = HiveServer2Hook()
- hook.get_records(sql)
-
- def test_multi_statements(self):
- from airflow.hooks.hive_hooks import HiveServer2Hook
- sqls = [
- "CREATE TABLE IF NOT EXISTS test_multi_statements (i INT)",
- "DROP TABLE test_multi_statements",
- ]
- hook = HiveServer2Hook()
- hook.get_records(sqls)
-
- def test_get_metastore_databases(self):
- if six.PY2:
- from airflow.hooks.hive_hooks import HiveMetastoreHook
- hook = HiveMetastoreHook()
- hook.get_databases()
-
- def test_to_csv(self):
- from airflow.hooks.hive_hooks import HiveServer2Hook
- sql = "select 1"
- hook = HiveServer2Hook()
- hook.to_csv(hql=sql, csv_filepath="/tmp/test_to_csv")
-
- def connect_mock(self, host, port,
- auth_mechanism, kerberos_service_name,
- user, database):
- self.assertEqual(database, self.nondefault_schema)
-
- @mock.patch('HiveServer2Hook.connect', return_value="foo")
- def test_select_conn_with_schema(self, connect_mock):
- from airflow.hooks.hive_hooks import HiveServer2Hook
-
- # Configure
- hook = HiveServer2Hook()
-
- # Run
- hook.get_conn(self.nondefault_schema)
-
- # Verify
- self.assertTrue(connect_mock.called)
- (args, kwargs) = connect_mock.call_args_list[0]
- self.assertEqual(self.nondefault_schema, kwargs['database'])
-
- def test_get_results_with_schema(self):
- from airflow.hooks.hive_hooks import HiveServer2Hook
- from unittest.mock import MagicMock
-
- # Configure
- sql = "select 1"
- schema = "notdefault"
- hook = HiveServer2Hook()
- cursor_mock = MagicMock(
- __enter__=cursor_mock,
- __exit__=None,
- execute=None,
- fetchall=[],
- )
- get_conn_mock = MagicMock(
- __enter__=get_conn_mock,
- __exit__=None,
- cursor=cursor_mock,
- )
- hook.get_conn = get_conn_mock
-
- # Run
- hook.get_results(sql, schema)
-
- # Verify
- get_conn_mock.assert_called_with(self.nondefault_schema)
-
- @mock.patch('HiveServer2Hook.get_results', return_value={'data': []})
- def test_get_records_with_schema(self, get_results_mock):
- from airflow.hooks.hive_hooks import HiveServer2Hook
-
- # Configure
- sql = "select 1"
- hook = HiveServer2Hook()
-
- # Run
- hook.get_records(sql, self.nondefault_schema)
-
- # Verify
- self.assertTrue(self.connect_mock.called)
- (args, kwargs) = self.connect_mock.call_args_list[0]
- self.assertEqual(sql, args[0])
- self.assertEqual(self.nondefault_schema, kwargs['schema'])
-
- @mock.patch('HiveServer2Hook.get_results', return_value={'data': []})
- def test_get_pandas_df_with_schema(self, get_results_mock):
- from airflow.hooks.hive_hooks import HiveServer2Hook
-
- # Configure
- sql = "select 1"
- hook = HiveServer2Hook()
-
- # Run
- hook.get_pandas_df(sql, self.nondefault_schema)
-
- # Verify
- self.assertTrue(self.connect_mock.called)
- (args, kwargs) = self.connect_mock.call_args_list[0]
- self.assertEqual(sql, args[0])
- self.assertEqual(self.nondefault_schema, kwargs['schema'])
-
class HivePrestoTest(HiveEnvironmentTest):
def test_hive(self):