You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@superset.apache.org by GitBox <gi...@apache.org> on 2018/11/08 05:50:37 UTC
[GitHub] timifasubaa closed pull request #6048: [WIP] Nosetest improvements
timifasubaa closed pull request #6048: [WIP] Nosetest improvements
URL: https://github.com/apache/incubator-superset/pull/6048
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/superset/cli.py b/superset/cli.py
index 6183601d02..05bf07ea43 100755
--- a/superset/cli.py
+++ b/superset/cli.py
@@ -140,55 +140,95 @@ def version(verbose):
def load_examples_run(load_test_data):
- print('Loading examples into {}'.format(db))
-
- data.load_css_templates()
+ #raise Exception("Where am I coming from?")
+ #import sys
+ #print(sys._getframe(1).f_code.co_name)
+ #print(sys._getframe(2).f_code.co_name)
+ #print(sys._getframe(3).f_code.co_name)
+ #print("^^^^")
+ #shave the time it spends loading and then parallelize the tests.
+
+ #do we need this?
+ # parallelize the loading...
+
+ #print('Loading examples into {}'.format(db))
+ #start_css_tmp = utils.now_as_float()
+ #data.load_css_templates()
+ #print("took {}".format(utils.now_as_float() - start_css_tmp))
print('Loading energy related dataset')
+ start_energy = utils.now_as_float()
data.load_energy()
+ print("took {}".format(utils.now_as_float() - start_energy))
print("Loading [World Bank's Health Nutrition and Population Stats]")
+ start_health = utils.now_as_float()
data.load_world_bank_health_n_pop()
+ energy_time = utils.now_as_float() - start_health
print('Loading [Birth names]')
+ start_birth = utils.now_as_float()
data.load_birth_names()
-
- print('Loading [Random time series data]')
- data.load_random_time_series_data()
-
- print('Loading [Random long/lat data]')
- data.load_long_lat_data()
-
- print('Loading [Country Map data]')
- data.load_country_map_data()
-
- print('Loading [Multiformat time series]')
- data.load_multiformat_time_series_data()
-
- print('Loading [Paris GeoJson]')
- data.load_paris_iris_geojson()
-
- print('Loading [San Francisco population polygons]')
- data.load_sf_population_polygons()
-
- print('Loading [Flights data]')
- data.load_flights()
-
- print('Loading [BART lines]')
- data.load_bart_lines()
-
- print('Loading [Multi Line]')
- data.load_multi_line()
-
- print('Loading [Misc Charts] dashboard')
- data.load_misc_dashboard()
-
- if load_test_data:
- print('Loading [Unicode test data]')
- data.load_unicode_test_data()
-
- print('Loading DECK.gl demo')
- data.load_deck_dash()
+ print(utils.now_as_float() - start_birth)
+
+ #print('Loading [Random time series data]')
+ #start_ts = utils.now_as_float()
+ #data.load_random_time_series_data()
+ #print("took {}".format(utils.now_as_float() - start_ts))
+
+ #print('Loading [Random long/lat data]')
+ #start_lola = utils.now_as_float()
+ #data.load_long_lat_data()
+ #print("took {}".format(utils.now_as_float() - start_lola))
+
+ #print('Loading [Country Map data]')
+ #start_map = utils.now_as_float()
+ #data.load_country_map_data()
+ #print("took {}".format(utils.now_as_float() - start_map))
+
+ #print('Loading [Multiformat time series]')
+ #start_mts = utils.now_as_float()
+ #data.load_multiformat_time_series_data()
+ #print("took {}".format(utils.now_as_float() - start_mts))
+
+ #print('Loading [Paris GeoJson]')
+ #start_geo = utils.now_as_float()
+ #data.load_paris_iris_geojson()
+ #print("took {}".format(utils.now_as_float() - start_geo))
+
+ #print('Loading [San Francisco population polygons]')
+ #start_polygons = utils.now_as_float()
+ #data.load_sf_population_polygons()
+ #print("took {}".format(utils.now_as_float() - start_polygons))
+
+ #print('Loading [Flights data]')
+ #start_flights = utils.now_as_float()
+ #data.load_flights()
+ #print("took {}".format(utils.now_as_float() - start_flights))
+
+ #print('Loading [BART lines]')
+ #start_bart = utils.now_as_float()
+ #data.load_bart_lines()
+ #print("took {}".format(utils.now_as_float() - start_bart))
+
+ #print('Loading [Multi Line]')
+ #start_ml = utils.now_as_float()
+ #data.load_multi_line()
+ #print("took {}".format(utils.now_as_float() - start_ml))
+
+ #print('Loading [Misc Charts] dashboard')
+ #start_misc = utils.now_as_float()
+ #data.load_misc_dashboard()
+ #print("took {}".format(utils.now_as_float() - start_misc))
+
+ #if load_test_data:
+ # print('Loading [Unicode test data]')
+ # data.load_unicode_test_data()
+
+ #print('Loading DECK.gl demo')
+ #start_deckgl = utils.now_as_float()
+ #data.load_deck_dash()
+ #print("took {}".format(utils.now_as_float() - start_deckgl))
@app.cli.command()
diff --git a/tests/access_tests.py b/tests/access_tests.py
index a669f5dec5..6864ccb6e2 100644
--- a/tests/access_tests.py
+++ b/tests/access_tests.py
@@ -87,6 +87,7 @@ def create_access_request(session, ds_type, ds_name, role_name, user_name):
class RequestAccessTests(SupersetTestCase):
requires_examples = False
+ #_multiprocess_shared_ = True
@classmethod
def setUpClass(cls):
@@ -170,10 +171,11 @@ def test_override_role_permissions_druid_and_table(self):
self.assertEquals(3, len(perms))
def test_override_role_permissions_drops_absent_perms(self):
+ #use energy usage instead?....
override_me = security_manager.find_role('override_me')
override_me.permissions.append(
security_manager.find_permission_view_menu(
- view_menu_name=self.get_table_by_name('long_lat').perm,
+ view_menu_name=self.get_table_by_name('energy_usage').perm,
permission_name='datasource_access'),
)
db.session.flush()
@@ -259,9 +261,9 @@ def test_clean_requests_after_db_grant(self):
gamma_user = security_manager.find_user(username='gamma')
access_request1 = create_access_request(
- session, 'table', 'long_lat', TEST_ROLE_1, 'gamma')
+ session, 'table', 'energy_usage', TEST_ROLE_1, 'gamma')
create_access_request(
- session, 'table', 'long_lat', TEST_ROLE_2, 'gamma2')
+ session, 'table', 'energy_usage', TEST_ROLE_2, 'gamma2')
ds_1_id = access_request1.datasource_id
# gamma gets granted database access
database = session.query(models.Database).first()
@@ -359,9 +361,9 @@ def test_approve(self, mock_send_mime):
# Case 2. Extend the role to have access to the table
access_request2 = create_access_request(
- session, 'table', 'long_lat', TEST_ROLE_NAME, 'gamma')
+ session, 'table', 'energy_usage', TEST_ROLE_NAME, 'gamma')
ds_2_id = access_request2.datasource_id
- long_lat_perm = access_request2.datasource.perm
+ energy_usage_perm = access_request2.datasource.perm
self.client.get(EXTEND_ROLE_REQUEST.format(
'table', access_request2.datasource_id, 'gamma', TEST_ROLE_NAME))
@@ -377,13 +379,13 @@ def test_approve(self, mock_send_mime):
'[Superset] Access to the datasource {} was granted'.format(
self.get_table(ds_2_id).full_name), call_args[2]['Subject'])
self.assertIn(TEST_ROLE_NAME, call_args[2].as_string())
- self.assertIn('long_lat', call_args[2].as_string())
+ self.assertIn('energy_usage', call_args[2].as_string())
# request was removed
self.assertFalse(access_requests)
- # table_role was extended to grant access to the long_lat table/
+ # table_role was extended to grant access to the energy_usage table/
perm_view = security_manager.find_permission_view_menu(
- 'datasource_access', long_lat_perm)
+ 'datasource_access', energy_usage_perm)
TEST_ROLE = security_manager.find_role(TEST_ROLE_NAME)
self.assertIn(perm_view, TEST_ROLE.permissions)
diff --git a/tests/base_tests.py b/tests/base_tests.py
index f69ab68381..7c707a496a 100644
--- a/tests/base_tests.py
+++ b/tests/base_tests.py
@@ -28,12 +28,14 @@ class SupersetTestCase(unittest.TestCase):
requires_examples = False
examples_loaded = False
+ _multiprocess_shared_ = True
+
def __init__(self, *args, **kwargs):
if (
self.requires_examples and
not os.environ.get('examples_loaded')
):
- logging.info('Loading examples')
+ logging.info('Loading examples in test case')
cli.load_examples_run(load_test_data=True)
logging.info('Done loading examples')
security_manager.sync_role_definitions()
diff --git a/tests/cache_tests.py b/tests/cache_tests.py
index 27a2c5f256..c4c0cada8b 100644
--- a/tests/cache_tests.py
+++ b/tests/cache_tests.py
@@ -13,6 +13,8 @@
class CacheTests(SupersetTestCase):
+ _multiprocess_shared_ = True
+
def __init__(self, *args, **kwargs):
super(CacheTests, self).__init__(*args, **kwargs)
diff --git a/tests/celery_tests.py b/tests/celery_tests.py
index 06b10031cd..ff16060320 100644
--- a/tests/celery_tests.py
+++ b/tests/celery_tests.py
@@ -37,6 +37,8 @@ class CeleryConfig(object):
class UtilityFunctionTests(SupersetTestCase):
+ _multiprocess_shared_ = True
+
# TODO(bkyryliuk): support more cases in CTA function.
def test_create_table_as(self):
q = SupersetQuery('SELECT * FROM outer_space;')
diff --git a/tests/core_tests.py b/tests/core_tests.py
index d0eeacf229..74401305f0 100644
--- a/tests/core_tests.py
+++ b/tests/core_tests.py
@@ -36,6 +36,7 @@
class CoreTests(SupersetTestCase):
requires_examples = True
+ _multiprocess_can_split_ = True
def __init__(self, *args, **kwargs):
super(CoreTests, self).__init__(*args, **kwargs)
@@ -378,7 +379,7 @@ def test_warm_up_cache(self):
data = self.get_json_resp(
'/superset/warm_up_cache?table_name=energy_usage&db_name=main')
- assert len(data) == 4
+ assert len(data) == 3
def test_shortner(self):
self.login(username='admin')
diff --git a/tests/dashboard_tests.py b/tests/dashboard_tests.py
index bc6c61f5b2..2b42bcbf18 100644
--- a/tests/dashboard_tests.py
+++ b/tests/dashboard_tests.py
@@ -19,6 +19,7 @@
class DashboardTests(SupersetTestCase):
requires_examples = True
+ _multiprocess_can_split_ = True
def __init__(self, *args, **kwargs):
super(DashboardTests, self).__init__(*args, **kwargs)
@@ -211,7 +212,7 @@ def test_add_slices(self, username='admin'):
dash = db.session.query(models.Dashboard).filter_by(
slug='births').first()
new_slice = db.session.query(models.Slice).filter_by(
- slice_name='Mapbox Long/Lat').first()
+ slice_name='Energy Force Layout').first()
existing_slice = db.session.query(models.Slice).filter_by(
slice_name='Name Cloud').first()
data = {
@@ -225,7 +226,7 @@ def test_add_slices(self, username='admin'):
dash = db.session.query(models.Dashboard).filter_by(
slug='births').first()
new_slice = db.session.query(models.Slice).filter_by(
- slice_name='Mapbox Long/Lat').first()
+ slice_name='Energy Force Layout').first()
assert new_slice in dash.slices
assert len(set(dash.slices)) == len(dash.slices)
@@ -233,7 +234,7 @@ def test_add_slices(self, username='admin'):
dash = db.session.query(models.Dashboard).filter_by(
slug='births').first()
dash.slices = [
- o for o in dash.slices if o.slice_name != 'Mapbox Long/Lat']
+ o for o in dash.slices if o.slice_name != 'Energy Force Layout']
db.session.commit()
def test_remove_slices(self, username='admin'):
diff --git a/tests/dataframe_test.py b/tests/dataframe_test.py
index b52b9508ab..4fce640b9e 100644
--- a/tests/dataframe_test.py
+++ b/tests/dataframe_test.py
@@ -12,6 +12,8 @@
class SupersetDataFrameTestCase(SupersetTestCase):
+ _multiprocess_can_split_ = True
+
def test_dedup(self):
self.assertEquals(
dedup(['foo', 'bar']),
diff --git a/tests/datasource_tests.py b/tests/datasource_tests.py
index 7da7d929f5..c7aa602305 100644
--- a/tests/datasource_tests.py
+++ b/tests/datasource_tests.py
@@ -14,6 +14,7 @@
class DatasourceTests(SupersetTestCase):
requires_examples = True
+ _multiprocess_can_split_ = True
def __init__(self, *args, **kwargs):
super(DatasourceTests, self).__init__(*args, **kwargs)
diff --git a/tests/db_engine_specs_test.py b/tests/db_engine_specs_test.py
index 81709dc67b..62ab713f9f 100644
--- a/tests/db_engine_specs_test.py
+++ b/tests/db_engine_specs_test.py
@@ -18,6 +18,8 @@
class DbEngineSpecsTestCase(SupersetTestCase):
+ _multiprocess_can_split_ = True
+
def test_0_progress(self):
log = """
17/02/07 18:26:27 INFO log.PerfLogger: <PERFLOG method=compile from=org.apache.hadoop.hive.ql.Driver>
diff --git a/tests/dict_import_export_tests.py b/tests/dict_import_export_tests.py
index c96ce63652..291ec3fcf6 100644
--- a/tests/dict_import_export_tests.py
+++ b/tests/dict_import_export_tests.py
@@ -26,6 +26,8 @@
class DictImportExportTests(SupersetTestCase):
"""Testing export import functionality for dashboards"""
+ _multiprocess_can_split_ = True
+
def __init__(self, *args, **kwargs):
super(DictImportExportTests, self).__init__(*args, **kwargs)
diff --git a/tests/druid_tests.py b/tests/druid_tests.py
index 00df0f8ea6..0fd07c9f5f 100644
--- a/tests/druid_tests.py
+++ b/tests/druid_tests.py
@@ -82,6 +82,7 @@ def __reduce__(self):
class DruidTests(SupersetTestCase):
"""Testing interactions with Druid"""
+ _multiprocess_shared_ = True
def __init__(self, *args, **kwargs):
super(DruidTests, self).__init__(*args, **kwargs)
diff --git a/tests/email_tests.py b/tests/email_tests.py
index 3ee9f3ebf2..4ca689554b 100644
--- a/tests/email_tests.py
+++ b/tests/email_tests.py
@@ -19,6 +19,9 @@
class EmailSmtpTest(unittest.TestCase):
+
+ #_multiprocess_shared_ = True
+
def setUp(self):
app.config['smtp_ssl'] = False
diff --git a/tests/model_tests.py b/tests/model_tests.py
index 1a78567964..0cb665931b 100644
--- a/tests/model_tests.py
+++ b/tests/model_tests.py
@@ -15,6 +15,7 @@
class DatabaseModelTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
def test_database_schema_presto(self):
sqlalchemy_uri = 'presto://presto.airbnb.io:8080/hive/default'
@@ -79,7 +80,7 @@ def test_database_impersonate_user(self):
def test_select_star(self):
main_db = get_main_database(db.session)
- table_name = 'bart_lines'
+ table_name = 'energy_usage'
sql = main_db.select_star(
table_name, show_cols=False, latest_partition=False)
expected = textwrap.dedent("""\
@@ -91,11 +92,10 @@ def test_select_star(self):
sql = main_db.select_star(
table_name, show_cols=True, latest_partition=False)
expected = textwrap.dedent("""\
- SELECT color,
- name,
- path_json,
- polyline
- FROM bart_lines
+ SELECT source,
+ target,
+ value
+ FROM energy_usage
LIMIT 100""".format(**locals()))
assert sql.startswith(expected)
@@ -129,6 +129,8 @@ def test_multi_statement(self):
class SqlaTableModelTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
+
def test_get_timestamp_expression(self):
tbl = self.get_table_by_name('birth_names')
diff --git a/tests/security_tests.py b/tests/security_tests.py
index fe7564ed14..d9435ba2dc 100644
--- a/tests/security_tests.py
+++ b/tests/security_tests.py
@@ -18,6 +18,8 @@ def get_perm_tuples(role_name):
class RolePermissionTests(SupersetTestCase):
"""Testing export import functionality for dashboards"""
+ _multiprocess_can_split_ = True
+
def __init__(self, *args, **kwargs):
super(RolePermissionTests, self).__init__(*args, **kwargs)
diff --git a/tests/sql_parse_tests.py b/tests/sql_parse_tests.py
index 3f66fedb43..fca1bc610c 100644
--- a/tests/sql_parse_tests.py
+++ b/tests/sql_parse_tests.py
@@ -8,8 +8,11 @@
from superset import sql_parse
+#can parallelize
+
class SupersetTestCase(unittest.TestCase):
+ _multiprocess_can_split_ = True
def extract_tables(self, query):
sq = sql_parse.SupersetQuery(query)
diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py
index a39643aa60..7019c97995 100644
--- a/tests/sqllab_tests.py
+++ b/tests/sqllab_tests.py
@@ -22,6 +22,8 @@
class SqlLabTests(SupersetTestCase):
"""Testings for Sql Lab"""
+ #_multiprocess_shared_ = True
+
def __init__(self, *args, **kwargs):
super(SqlLabTests, self).__init__(*args, **kwargs)
diff --git a/tests/utils_tests.py b/tests/utils_tests.py
index eb164b8b1a..2ca50bc084 100644
--- a/tests/utils_tests.py
+++ b/tests/utils_tests.py
@@ -73,6 +73,9 @@ def mock_to_adhoc(filt, expressionType='SIMPLE', clause='where'):
class UtilsTestCase(unittest.TestCase):
+
+ _multiprocess_can_split_ = True
+
def test_json_int_dttm_ser(self):
dttm = datetime(2020, 1, 1)
ts = 1577836800000.0
diff --git a/tests/viz_tests.py b/tests/viz_tests.py
index da7c02f940..48110105c2 100644
--- a/tests/viz_tests.py
+++ b/tests/viz_tests.py
@@ -20,6 +20,8 @@
class BaseVizTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
+
def test_constructor_exception_no_datasource(self):
form_data = {}
datasource = None
@@ -172,6 +174,7 @@ def test_cache_timeout(self):
class TableVizTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
def test_get_data_applies_percentage(self):
form_data = {
@@ -396,6 +399,8 @@ def test_should_be_timeseries_raises_when_no_granularity(self):
class PairedTTestTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
+
def test_get_data_transforms_dataframe(self):
form_data = {
'groupby': ['groupA', 'groupB', 'groupC'],
@@ -528,6 +533,7 @@ def test_get_data_empty_null_keys(self):
class PartitionVizTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
@patch('superset.viz.BaseViz.query_obj')
def test_query_obj_time_series_option(self, super_query_obj):
@@ -753,6 +759,7 @@ def test_get_data_calls_correct_method(self):
class RoseVisTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
def test_rose_vis_get_data(self):
raw = {}
@@ -793,6 +800,7 @@ def test_rose_vis_get_data(self):
class TimeSeriesTableVizTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
def test_get_data_metrics(self):
form_data = {
@@ -873,6 +881,7 @@ def test_query_obj_throws_metrics_and_groupby(self, super_query_obj):
class BaseDeckGLVizTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
def test_get_metrics(self):
form_data = load_fixture('deck_path_form_data.json')
@@ -1066,6 +1075,8 @@ def test_filter_nulls(self, mock_uuid4):
class TimeSeriesVizTestCase(SupersetTestCase):
+ _multiprocess_shared_ = True
+
def test_timeseries_unicode_data(self):
datasource = self.get_datasource_mock()
form_data = {
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: notifications-unsubscribe@superset.apache.org
For additional commands, e-mail: notifications-help@superset.apache.org