You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by do...@apache.org on 2020/01/16 02:52:32 UTC
[madlib] branch master updated (515dc25 -> 5ddec47)
This is an automated email from the ASF dual-hosted git repository.
domino pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git.
from 515dc25 misc user doc clarifications
new dbae722 DL: Fix metrics_elapsed_time for fit multi model
new 5ddec47 DL: Drop leftover tables
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../modules/deep_learning/input_data_preprocessor.py_in | 4 +++-
.../deep_learning/madlib_keras_fit_multiple_model.py_in | 5 +++--
.../postgres/modules/deep_learning/madlib_keras_helper.py_in | 10 +++++-----
.../deep_learning/test/madlib_keras_model_selection.sql_in | 4 ++++
.../deep_learning/test/unit_tests/test_madlib_keras.py_in | 4 +++-
5 files changed, 18 insertions(+), 9 deletions(-)
[madlib] 02/02: DL: Drop leftover tables
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
domino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit 5ddec47242415b32f8b26849c73e70f3a4a20448
Author: Nikhil Kak <nk...@pivotal.io>
AuthorDate: Tue Jan 14 11:12:33 2020 -0800
DL: Drop leftover tables
JIRA: MADLIB-1404
While calling the function `get_accessible_gpus_for_seg`, we create an
intermediate gpu table by calling the madlib function gpu_configuration.
This table was not getting dropped at the end.
This commit drops the intermediate gpu table created in
get_accessible_gpus_for_seg.
Co-authored-by: Ekta Khanna <ek...@pivotal.io>
---
.../modules/deep_learning/input_data_preprocessor.py_in | 4 +++-
.../postgres/modules/deep_learning/madlib_keras_helper.py_in | 10 +++++-----
.../deep_learning/test/unit_tests/test_madlib_keras.py_in | 4 +++-
3 files changed, 11 insertions(+), 7 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
index 3a5f118..605d439 100644
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
+++ b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
@@ -328,6 +328,8 @@ class InputDataPreprocessorDL(object):
all_segments = False
if self.distribution_rules == 'gpu_segments':
+ #TODO can we reuse the function `get_accessible_gpus_for_seg` from
+ # madlib_keras_helper
gpu_info_table = unique_string(desp='gpu_info')
plpy.execute("""
SELECT {self.schema_madlib}.gpu_configuration('{gpu_info_table}')
@@ -339,6 +341,7 @@ class InputDataPreprocessorDL(object):
gpu_query_result = plpy.execute(gpu_query)[0]['gpu_config']
if not gpu_query_result:
plpy.error("{self.module_name}: No GPUs configured on hosts.".format(self=self))
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
gpu_config_hostnames = "ARRAY{0}".format(gpu_query_result)
# find hosts with gpus
@@ -351,7 +354,6 @@ class InputDataPreprocessorDL(object):
AND hostname=ANY({gpu_config_hostnames})
""".format(**locals())
segment_ids_result = plpy.execute(get_segment_query)[0]
- plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
self.gpu_config = "ARRAY{0}".format(sorted(segment_ids_result['segment_ids']))
self.distribution_rules = "ARRAY{0}".format(sorted(segment_ids_result['dbid']))
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
index 6e006d5..5be078b 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
@@ -269,24 +269,24 @@ def create_summary_view(module_name, model_table, mst_key):
""".format(**locals()))
return tmp_view_summary
-def get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name):
+def get_accessible_gpus_for_seg(schema_madlib, segments_per_host, module_name):
if is_platform_pg():
gpus = GPUInfoFunctions.get_gpu_info_from_tensorflow()
if not gpus:
plpy.error("{0} error: No GPUs configured on host.".format(module_name))
return [len(gpus)]
else:
- gpu_table_name = unique_string(desp = 'gpu_table')
+ gpu_info_table = unique_string(desp = 'gpu_info')
gpu_table_query = """
- SELECT {schema_madlib}.gpu_configuration('{gpu_table_name}')
+ SELECT {schema_madlib}.gpu_configuration('{gpu_info_table}')
""".format(**locals())
plpy.execute(gpu_table_query)
gpu_query = """
- SELECT hostname, count(*) AS count FROM {gpu_table_name} GROUP BY hostname
+ SELECT hostname, count(*) AS count FROM {gpu_info_table} GROUP BY hostname
""".format(**locals())
gpu_query_result = plpy.execute(gpu_query)
-
+ plpy.execute("DROP TABLE IF EXISTS {0}".format(gpu_info_table))
if not gpu_query_result:
plpy.error("{0} error: No GPUs configured on hosts.".format(module_name))
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
index a8bb629..3714de5 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras.py_in
@@ -1336,6 +1336,7 @@ class MadlibKerasHelperTestCase(unittest.TestCase):
self.plpy_mock_execute.side_effect = \
[ [],
[ {'hostname': 'mdw0', 'count' : 1}],
+ [],
[ {'hostname': 'mdw0', 'segment_id' : 0},
{'hostname': 'mdw1', 'segment_id' : 1},
{'hostname': 'mdw2', 'segment_id' : 2}
@@ -1351,6 +1352,7 @@ class MadlibKerasHelperTestCase(unittest.TestCase):
self.plpy_mock_execute.side_effect = \
[[],
[ {'hostname': 'mdw0', 'count' : 1}],
+ [],
[ {'hostname': 'mdw0', 'segment_id' : 0},
{'hostname': 'mdw0', 'segment_id' : 1},
{'hostname': 'mdw1', 'segment_id' : 2},
@@ -1364,7 +1366,7 @@ class MadlibKerasHelperTestCase(unittest.TestCase):
self.subject.is_platform_pg = Mock(return_value = False)
- self.plpy_mock_execute.side_effect = [[],[]]
+ self.plpy_mock_execute.side_effect = [[],[],[]]
with self.assertRaises(plpy.PLPYException) as error:
self.subject.get_accessible_gpus_for_seg('schema_madlib', 2, 'foo')
self.assertIn('no gpus configured on hosts', str(error.exception).lower())
[madlib] 01/02: DL: Fix metrics_elapsed_time for fit multi model
Posted by do...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
domino pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit dbae72239ed3d201bf675e7aafd6f44903f6f007
Author: Ekta Khanna <ek...@pivotal.io>
AuthorDate: Mon Jan 13 15:48:11 2020 -0800
DL: Fix metrics_elapsed_time for fit multi model
JIRA: MADLIB-1403
Prior to this commit, for madlib_keras_fit_multiple_model(), the
metrics_elapsed_time in info table was incorrectly stored as the per
iteration evaluation time instead of the elapsed time. This commit
fixes that issue and adds an assert for it too.
Co-authored-by: Nikhil Kak <nk...@pivotal.io>
---
.../modules/deep_learning/madlib_keras_fit_multiple_model.py_in | 5 +++--
.../modules/deep_learning/test/madlib_keras_model_selection.sql_in | 4 ++++
2 files changed, 7 insertions(+), 2 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
index 273321e..ae577e8 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
@@ -172,6 +172,7 @@ class FitMultipleModel():
# WARNING: set orca off to prevent unwanted redistribution
with OptimizerControl(False):
self.start_training_time = datetime.datetime.now()
+ self.metrics_elapsed_start_time = time.time()
self.train_multiple_model()
self.end_training_time = datetime.datetime.now()
self.insert_info_table()
@@ -219,7 +220,7 @@ class FitMultipleModel():
weights = query_weights(self.model_output_table, self.model_weights_col,
self.mst_key_col, mst[self.mst_key_col])
model_arch, _ = get_model_arch_weights(self.model_arch_table, mst[self.model_id_col])
- metric_eval_time, metric, loss = compute_loss_and_metrics(
+ _, metric, loss = compute_loss_and_metrics(
self.schema_madlib, table, "$madlib${0}$madlib$".format(
mst[self.compile_params_col]),
model_arch,
@@ -230,7 +231,7 @@ class FitMultipleModel():
images_per_seg,
[], [], epoch, True)
mst_metric_eval_time[mst[self.mst_key_col]] \
- .append(metric_eval_time)
+ .append(time.time() - self.metrics_elapsed_start_time)
mst_loss[mst[self.mst_key_col]].append(loss)
mst_metric[mst[self.mst_key_col]].append(metric)
self.info_str += "\n\tmst_key={0}: metric={1}, loss={2}".format(mst[self.mst_key_col], metric, loss)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
index c36d48f..26c1a34 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_model_selection.sql_in
@@ -320,6 +320,10 @@ SELECT assert(
'Keras Fit Multiple Output Info Validation failed. Actual:' || __to_char(info))
FROM (SELECT * FROM iris_multiple_model_info) info;
+SELECT assert(metrics_elapsed_time[3] - metrics_elapsed_time[1] > 0,
+ 'Keras Fit Multiple invalid elapsed time calculation.')
+FROM (SELECT * FROM iris_multiple_model_info) info;
+
SELECT assert(
name = 'multi_model_name' AND
description = 'multi_model_descr' AND