You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by kh...@apache.org on 2020/10/27 20:18:07 UTC
[madlib] 02/08: DL: [AutoML] Hyperopt implementation
This is an automated email from the ASF dual-hosted git repository.
khannaekta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
commit f2b80f211cf5462782a9238dfac9f41d37332c6a
Author: Advitya Gemawat <ag...@vmware.com>
AuthorDate: Sun Sep 20 21:58:36 2020 -0700
DL: [AutoML] Hyperopt implementation
JIRA: MADLIB-1453
We expand AutoML capabilities in MADlib by adding support for Hyperopt,
another hyperparameter optimization method to minimize losses with
awkward search spaces. The user can declaratively specify the automl
method name with the related automl params (with all other argument
declarations staying the same across different AutoML methods) and our
API handles the exploration and execution components with the algorithm
workload info displayed to the user.
We implement Hyperopt for Massively Parallel Processing (MPP) to perform
model selection on top of our existing Model hOpper (MOP) infrastructure
for model training/evaluation. Our API currently offers declarative
support for Random Search and Tree-structured Parzen Estimator (TPE), a
Bayesian Optimization like approach.
Additionally,
- The user will have to install the hyperopt python module on the master
host when calling hyperopt automl_method.
- Print 'best-so-far' console info for each AutoML method (i.e.
Hyperband and Hyperopt)
- Disable automl method argument prefixing (while still retaining case
insensitivity).
- If no automl_method is passed in, the default is `hyperband`.
- Default values for automl_params:
Hyperband: `R=6, eta=3, skip_last=0`
Hyperopt: `num_configs=20, num_iterations=5, algorithm=tpe`
- For hyperopt, the metrics_elapsed_time in the output info table is
accumulated across each trial.
Co-authored-by: Nikhil Kak <nk...@vmware.com>
Co-authored-by: Ekta Khanna <ek...@vmware.com>
---
.../deep_learning/input_data_preprocessor.py_in | 3 +-
.../deep_learning/madlib_keras_automl.py_in | 839 +++++++++++++++++----
.../deep_learning/madlib_keras_automl.sql_in | 29 +-
.../madlib_keras_fit_multiple_model.py_in | 24 +-
.../deep_learning/madlib_keras_helper.py_in | 57 +-
.../madlib_keras_model_selection.py_in | 63 +-
.../deep_learning/madlib_keras_validator.py_in | 2 +-
.../deep_learning/test/madlib_keras_automl.sql_in | 459 +++++++----
.../test/unit_tests/test_madlib_keras_automl.py_in | 74 ++
.../test_madlib_keras_model_selection_table.py_in | 9 +-
.../postgres/modules/utilities/utilities.py_in | 7 +
11 files changed, 1181 insertions(+), 385 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
index 605d439..1d395a6 100644
--- a/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
+++ b/src/ports/postgres/modules/deep_learning/input_data_preprocessor.py_in
@@ -610,7 +610,7 @@ class InputDataPreprocessorDL(object):
{self.buffer_size} AS buffer_size,
{self.normalizing_const}::{FLOAT32_SQL_TYPE} AS {normalizing_const_colname},
{self.num_classes} AS {num_classes_colname},
- {self.distribution_rules} AS distribution_rules,
+ {self.distribution_rules} AS {distribution_rules},
{self.gpu_config} AS {internal_gpu_config}
""".format(self=self, class_level_str=class_level_str,
dependent_varname_colname=DEPENDENT_VARNAME_COLNAME,
@@ -620,6 +620,7 @@ class InputDataPreprocessorDL(object):
normalizing_const_colname=NORMALIZING_CONST_COLNAME,
num_classes_colname=NUM_CLASSES_COLNAME,
internal_gpu_config=INTERNAL_GPU_CONFIG,
+ distribution_rules=DISTRIBUTION_RULES,
FLOAT32_SQL_TYPE=FLOAT32_SQL_TYPE)
plpy.execute(query)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in
index 0f71fdf..d6eeba3 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.py_in
@@ -17,28 +17,35 @@
# specific language governing permissions and limitations
# under the License.
+from ast import literal_eval
from datetime import datetime
-import plpy
+from hyperopt import hp, rand, tpe, atpe, Trials, STATUS_OK, STATUS_RUNNING
+from hyperopt.base import Domain
import math
-from time import time
+import numpy as np
+import plpy
+import time
from madlib_keras_validator import MstLoaderInputValidator
-from utilities.utilities import unique_string, add_postfix, extract_keyvalue_params, \
- _assert, _assert_equal, rename_table
+# from utilities.admin import cleanup_madlib_temp_tables
+from utilities.utilities import get_current_timestamp, get_seg_number, get_segments_per_host, \
+ unique_string, add_postfix, extract_keyvalue_params, _assert, _assert_equal, rename_table
from utilities.control import MinWarning, SetGUC
from madlib_keras_fit_multiple_model import FitMultipleModel
+from madlib_keras_helper import generate_row_string
+from madlib_keras_helper import DISTRIBUTION_RULES
from madlib_keras_model_selection import MstSearch, ModelSelectionSchema
from keras_model_arch_table import ModelArchSchema
-from utilities.validate_args import table_exists, drop_tables
+from utilities.validate_args import table_exists, drop_tables, input_tbl_valid
from utilities.validate_args import quote_ident
-
-class AutoMLSchema:
+class AutoMLConstants:
BRACKET = 's'
ROUND = 'i'
CONFIGURATIONS = 'n_i'
RESOURCES = 'r_i'
HYPERBAND = 'hyperband'
+ HYPEROPT = 'hyperopt'
R = 'R'
ETA = 'eta'
SKIP_LAST = 'skip_last'
@@ -47,7 +54,12 @@ class AutoMLSchema:
TEMP_MST_SUMMARY_TABLE = add_postfix(TEMP_MST_TABLE, '_summary')
TEMP_OUTPUT_TABLE = unique_string('temp_output_table')
METRICS_ITERS = 'metrics_iters' # custom column
-
+ NUM_CONFIGS = 'num_configs'
+ NUM_ITERS = 'num_iterations'
+ ALGORITHM = 'algorithm'
+ TIME_FORMAT = '%Y-%m-%d %H:%M:%S'
+ INT_MAX = 2 ** 31 - 1
+ TARGET_SCHEMA = 'public'
@MinWarning("warning")
class HyperbandSchedule():
@@ -66,6 +78,7 @@ class HyperbandSchedule():
self.R = R # maximum iterations/epochs allocated to a configuration
self.eta = eta # defines downsampling rate
self.skip_last = skip_last
+ self.module_name = 'hyperband_schedule'
self.validate_inputs()
# number of unique executions of Successive Halving (minus one)
@@ -87,12 +100,12 @@ class HyperbandSchedule():
"""
Validates user input values
"""
- _assert(self.eta > 1, "DL: eta must be greater than 1")
- _assert(self.R >= self.eta, "DL: R should not be less than eta")
+ _assert(self.eta > 1, "{0}: eta must be greater than 1".format(self.module_name))
+ _assert(self.R >= self.eta, "{0}: R should not be less than eta".format(self.module_name))
def validate_s_max(self):
- _assert(self.skip_last >= 0 and self.skip_last < self.s_max+1, "DL: skip_last must be " +
- "non-negative and less than {0}".format(self.s_max))
+ _assert(self.skip_last >= 0 and self.skip_last < self.s_max+1, "{0}: skip_last must be " +
+ "non-negative and less than {1}".format(self.module_name,self.s_max))
def calculate_schedule(self):
"""
@@ -108,10 +121,10 @@ class HyperbandSchedule():
n_i = n*math.pow(self.eta, -i)
r_i = r*math.pow(self.eta, i)
- self.schedule_vals.append({AutoMLSchema.BRACKET: s,
- AutoMLSchema.ROUND: i,
- AutoMLSchema.CONFIGURATIONS: int(n_i),
- AutoMLSchema.RESOURCES: int(round(r_i))})
+ self.schedule_vals.append({AutoMLConstants.BRACKET: s,
+ AutoMLConstants.ROUND: i,
+ AutoMLConstants.CONFIGURATIONS: int(n_i),
+ AutoMLConstants.RESOURCES: int(round(r_i))})
def create_schedule_table(self):
"""Initializes the output schedule table"""
@@ -124,20 +137,19 @@ class HyperbandSchedule():
unique ({s}, {i})
);
""".format(self=self,
- s=AutoMLSchema.BRACKET,
- i=AutoMLSchema.ROUND,
- n_i=AutoMLSchema.CONFIGURATIONS,
- r_i=AutoMLSchema.RESOURCES)
- with MinWarning('warning'):
- plpy.execute(create_query)
+ s=AutoMLConstants.BRACKET,
+ i=AutoMLConstants.ROUND,
+ n_i=AutoMLConstants.CONFIGURATIONS,
+ r_i=AutoMLConstants.RESOURCES)
+ plpy.execute(create_query)
def insert_into_schedule_table(self):
"""Insert everything in self.schedule_vals into the output schedule table."""
for sd in self.schedule_vals:
- sd_s = sd[AutoMLSchema.BRACKET]
- sd_i = sd[AutoMLSchema.ROUND]
- sd_n_i = sd[AutoMLSchema.CONFIGURATIONS]
- sd_r_i = sd[AutoMLSchema.RESOURCES]
+ sd_s = sd[AutoMLConstants.BRACKET]
+ sd_i = sd[AutoMLConstants.ROUND]
+ sd_n_i = sd[AutoMLConstants.CONFIGURATIONS]
+ sd_r_i = sd[AutoMLConstants.RESOURCES]
insert_query = """
INSERT INTO
{self.schedule_table}(
@@ -152,27 +164,28 @@ class HyperbandSchedule():
{sd_n_i},
{sd_r_i}
)
- """.format(s_col=AutoMLSchema.BRACKET,
- i_col=AutoMLSchema.ROUND,
- n_i_col=AutoMLSchema.CONFIGURATIONS,
- r_i_col=AutoMLSchema.RESOURCES,
+ """.format(s_col=AutoMLConstants.BRACKET,
+ i_col=AutoMLConstants.ROUND,
+ n_i_col=AutoMLConstants.CONFIGURATIONS,
+ r_i_col=AutoMLConstants.RESOURCES,
**locals())
plpy.execute(insert_query)
-@MinWarning("warning")
-class KerasAutoML():
- """The core AutoML function for running AutoML algorithms such as Hyperband.
- This function executes the hyperband rounds 'diagonally' to evaluate multiple configurations together
- and leverage the compute power of MPP databases such as Greenplum.
+# @MinWarning("warning")
+class KerasAutoML(object):
+ """
+ The core AutoML class for running AutoML algorithms such as Hyperband and Hyperopt.
"""
def __init__(self, schema_madlib, source_table, model_output_table, model_arch_table, model_selection_table,
model_id_list, compile_params_grid, fit_params_grid, automl_method='hyperband',
- automl_params='R=6, eta=3, skip_last=0', random_state=None, object_table=None,
+ automl_params=None, random_state=None, object_table=None,
use_gpus=False, validation_table=None, metrics_compute_frequency=None,
name=None, description=None, **kwargs):
self.schema_madlib = schema_madlib
self.source_table = source_table
self.model_output_table = model_output_table
+ self.module_name = 'madlib_keras_automl'
+ input_tbl_valid(self.source_table, self.module_name)
if self.model_output_table:
self.model_info_table = add_postfix(self.model_output_table, '_info')
self.model_summary_table = add_postfix(self.model_output_table, '_summary')
@@ -199,10 +212,9 @@ class KerasAutoML():
module_name='madlib_keras_automl'
)
- self.automl_method = automl_method if automl_method else 'hyperband'
- self.automl_params = automl_params if automl_params else 'R=6, eta=3, skip_last=0'
+ self.automl_method = automl_method
+ self.automl_params = automl_params
self.random_state = random_state
- self.validate_and_define_inputs()
self.object_table = object_table
self.use_gpus = use_gpus if use_gpus else False
@@ -212,13 +224,7 @@ class KerasAutoML():
self.description = description
if self.validation_table:
- AutoMLSchema.LOSS_METRIC = 'validation_loss_final'
-
- self.create_model_output_table()
- self.create_model_output_info_table()
-
- if AutoMLSchema.HYPERBAND.startswith(self.automl_method.lower()):
- self.find_hyperband_config()
+ AutoMLConstants.LOSS_METRIC = 'validation_loss_final'
def create_model_output_table(self):
output_table_create_query = """
@@ -228,8 +234,8 @@ class KerasAutoML():
{ModelArchSchema.MODEL_ARCH} JSON)
""".format(self=self, ModelSelectionSchema=ModelSelectionSchema,
ModelArchSchema=ModelArchSchema)
- with MinWarning('warning'):
- plpy.execute(output_table_create_query)
+ # with MinWarning('warning'):
+ plpy.execute(output_table_create_query)
def create_model_output_info_table(self):
info_table_create_query = """
@@ -252,38 +258,87 @@ class KerasAutoML():
validation_metrics DOUBLE PRECISION[],
validation_loss DOUBLE PRECISION[],
{AutoMLSchema.METRICS_ITERS} INTEGER[])
- """.format(self=self, ModelSelectionSchema=ModelSelectionSchema,
- ModelArchSchema=ModelArchSchema, AutoMLSchema=AutoMLSchema)
- with MinWarning('warning'):
- plpy.execute(info_table_create_query)
+ """.format(self=self,
+ ModelSelectionSchema=ModelSelectionSchema,
+ ModelArchSchema=ModelArchSchema,
+ AutoMLSchema=AutoMLConstants)
+ plpy.execute(info_table_create_query)
- def validate_and_define_inputs(self):
+ def update_model_selection_table(self):
+ """
+ Drops and re-create the mst table to only include the best performing model configuration.
+ """
+ drop_tables([self.model_selection_table])
- if AutoMLSchema.HYPERBAND.startswith(self.automl_method.lower()):
- automl_params_dict = extract_keyvalue_params(self.automl_params,
- default_values={'R': 6, 'eta': 3, 'skip_last': 0},
- lower_case_names=False)
- # casting dict values to int
- for i in automl_params_dict:
- automl_params_dict[i] = int(automl_params_dict[i])
- _assert(len(automl_params_dict) >= 1 or len(automl_params_dict) <= 3,
- "DL: Only R, eta, and skip_last may be specified")
- for i in automl_params_dict:
- if i == AutoMLSchema.R:
- self.R = automl_params_dict[AutoMLSchema.R]
- elif i == AutoMLSchema.ETA:
- self.eta = automl_params_dict[AutoMLSchema.ETA]
- elif i == AutoMLSchema.SKIP_LAST:
- self.skip_last = automl_params_dict[AutoMLSchema.SKIP_LAST]
- else:
- plpy.error("DL: {0} is an invalid param".format(i))
- _assert(self.eta > 1, "DL: eta must be greater than 1")
- _assert(self.R >= self.eta, "DL: R should not be less than eta")
- self.s_max = int(math.floor(math.log(self.R, self.eta)))
- _assert(self.skip_last >= 0 and self.skip_last < self.s_max+1, "DL: skip_last must be " +
- "non-negative and less than {0}".format(self.s_max))
- else:
- plpy.error("DL: Only hyperband is currently supported as the automl method")
+ # only retaining best performing config
+ plpy.execute("CREATE TABLE {self.model_selection_table} AS SELECT {ModelSelectionSchema.MST_KEY}, " \
+ "{ModelSelectionSchema.MODEL_ID}, {ModelSelectionSchema.COMPILE_PARAMS}, " \
+ "{ModelSelectionSchema.FIT_PARAMS} FROM {self.model_info_table} " \
+ "ORDER BY {AutoMLSchema.LOSS_METRIC} LIMIT 1".format(self=self,
+ AutoMLSchema=AutoMLConstants,
+ ModelSelectionSchema=ModelSelectionSchema))
+
+ def generate_model_output_summary_table(self, model_training):
+ """
+ Creates and populates static values related to the AutoML workload.
+ :param model_training: Fit Multiple function call object.
+ """
+ #TODO this code is duplicated in create_model_summary_table
+ name = 'NULL' if self.name is None else '$MAD${0}$MAD$'.format(self.name)
+ descr = 'NULL' if self.description is None else '$MAD${0}$MAD$'.format(self.description)
+ object_table = 'NULL' if self.object_table is None else '$MAD${0}$MAD$'.format(self.object_table)
+ random_state = 'NULL' if self.random_state is None else '$MAD${0}$MAD$'.format(self.random_state)
+ validation_table = 'NULL' if self.validation_table is None else '$MAD${0}$MAD$'.format(self.validation_table)
+
+ create_query = plpy.prepare("""
+ CREATE TABLE {self.model_summary_table} AS
+ SELECT
+ $MAD${self.source_table}$MAD$::TEXT AS source_table,
+ {validation_table}::TEXT AS validation_table,
+ $MAD${self.model_output_table}$MAD$::TEXT AS model,
+ $MAD${self.model_info_table}$MAD$::TEXT AS model_info,
+ (SELECT dependent_varname FROM {model_training.model_summary_table})
+ AS dependent_varname,
+ (SELECT independent_varname FROM {model_training.model_summary_table})
+ AS independent_varname,
+ $MAD${self.model_arch_table}$MAD$::TEXT AS model_arch_table,
+ $MAD${self.model_selection_table}$MAD$::TEXT AS model_selection_table,
+ $MAD${self.automl_method}$MAD$::TEXT AS automl_method,
+ $MAD${self.automl_params}$MAD$::TEXT AS automl_params,
+ {random_state}::TEXT AS random_state,
+ {object_table}::TEXT AS object_table,
+ {self.use_gpus} AS use_gpus,
+ (SELECT metrics_compute_frequency FROM {model_training.model_summary_table})::INTEGER
+ AS metrics_compute_frequency,
+ {name}::TEXT AS name,
+ {descr}::TEXT AS description,
+ '{self.start_training_time}'::TIMESTAMP AS start_training_time,
+ '{self.end_training_time}'::TIMESTAMP AS end_training_time,
+ (SELECT madlib_version FROM {model_training.model_summary_table}) AS madlib_version,
+ (SELECT num_classes FROM {model_training.model_summary_table})::INTEGER AS num_classes,
+ (SELECT class_values FROM {model_training.model_summary_table}) AS class_values,
+ (SELECT dependent_vartype FROM {model_training.model_summary_table})
+ AS dependent_vartype,
+ (SELECT normalizing_const FROM {model_training.model_summary_table})
+ AS normalizing_const
+ """.format(self=self,
+ validation_table=validation_table,
+ random_state=random_state,
+ object_table=object_table,
+ name=name,
+ descr=descr,
+ model_training=model_training))
+
+ # with MinWarning('warning'):
+ plpy.execute(create_query)
+
+ def is_automl_method(self, method_name):
+ """
+ Utility function to check automl method name.
+ :param method_name: name of chosen method name to check.
+ :return: boolean
+ """
+ return self.automl_method.lower() == method_name.lower()
def _is_valid_metrics_compute_frequency(self, num_iterations):
"""
@@ -296,9 +351,97 @@ class KerasAutoML():
(self.metrics_compute_frequency >= 1 and \
self.metrics_compute_frequency <= num_iterations)
+ def print_best_mst_so_far(self):
+ """
+ Prints mst keys with best train/val losses at a given point.
+ """
+ best_so_far = '\n'
+ best_so_far += self.print_best_helper('training')
+ if self.validation_table:
+ best_so_far += self.print_best_helper('validation')
+ plpy.info(best_so_far)
+
+ def print_best_helper(self, keyword):
+ """
+ Helper function to Prints mst keys with best train/val losses at a given point.
+ :param keyword: column prefix ('training' or 'validation')
+ :return:
+ """
+ metrics_word, loss_word = keyword + '_metrics_final', keyword + '_loss_final'
+
+ res_str = 'Best {keyword} loss so far:\n'.format(keyword=keyword)
+ best_value = plpy.execute("SELECT {ModelSelectionSchema.MST_KEY}, {metrics_word}, " \
+ "{loss_word} FROM {self.model_info_table} ORDER BY " \
+ "{loss_word} LIMIT 1".format(self=self, ModelSelectionSchema=ModelSelectionSchema,
+ metrics_word=metrics_word, loss_word=loss_word))[0]
+ mst_key_value, metric_value, loss_value = best_value[ModelSelectionSchema.MST_KEY], \
+ best_value[metrics_word], best_value[loss_word]
+ res_str += ModelSelectionSchema.MST_KEY + '=' + str(mst_key_value) + ': metric=' + str(metric_value) + \
+ ', loss=' + str(loss_value) + '\n'
+ return res_str
+
+ def remove_temp_tables(self, model_training):
+ """
+ Remove all intermediate tables created for AutoML runs/updates.
+ :param model_training: Fit Multiple function call object.
+ """
+ drop_tables([model_training.original_model_output_table, model_training.model_info_table,
+ model_training.model_summary_table, AutoMLConstants.TEMP_MST_TABLE,
+ AutoMLConstants.TEMP_MST_SUMMARY_TABLE])
+
+# @MinWarning("warning")
+class AutoMLHyperband(KerasAutoML):
+ """
+ This class implements Hyperband, an infinite-arm bandit based algorithm that speeds up random search
+ through adaptive resource allocation, successive halving (SHA), and early stopping.
+
+ This class showcases a novel hyperband implementation by executing the hyperband rounds 'diagonally'
+ to evaluate multiple configurations together and leverage the compute power of MPP databases such as Greenplum.
+
+ This automl method inherits qualities from the automl class.
+ """
+ def __init__(self, schema_madlib, source_table, model_output_table, model_arch_table, model_selection_table,
+ model_id_list, compile_params_grid, fit_params_grid, automl_method,
+ automl_params, random_state=None, object_table=None,
+ use_gpus=False, validation_table=None, metrics_compute_frequency=None,
+ name=None, description=None, **kwargs):
+ automl_method = automl_method if automl_method else AutoMLConstants.HYPERBAND
+ automl_params = automl_params if automl_params else 'R=6, eta=3, skip_last=0'
+ KerasAutoML.__init__(self, schema_madlib, source_table, model_output_table, model_arch_table,
+ model_selection_table, model_id_list, compile_params_grid, fit_params_grid,
+ automl_method, automl_params, random_state, object_table, use_gpus,
+ validation_table, metrics_compute_frequency, name, description, **kwargs)
+ self.validate_and_define_inputs()
+ self.create_model_output_table()
+ self.create_model_output_info_table()
+ self.find_hyperband_config()
+
+ def validate_and_define_inputs(self):
+ automl_params_dict = extract_keyvalue_params(self.automl_params,
+ lower_case_names=False)
+ # casting dict values to int
+ for i in automl_params_dict:
+ automl_params_dict[i] = int(automl_params_dict[i])
+ _assert(len(automl_params_dict) >= 1 and len(automl_params_dict) <= 3,
+ "{0}: Only R, eta, and skip_last may be specified".format(self.module_name))
+ for i in automl_params_dict:
+ if i == AutoMLConstants.R:
+ self.R = automl_params_dict[AutoMLConstants.R]
+ elif i == AutoMLConstants.ETA:
+ self.eta = automl_params_dict[AutoMLConstants.ETA]
+ elif i == AutoMLConstants.SKIP_LAST:
+ self.skip_last = automl_params_dict[AutoMLConstants.SKIP_LAST]
+ else:
+ plpy.error("{0}: {1} is an invalid automl param".format(self.module_name, i))
+ _assert(self.eta > 1, "{0}: eta must be greater than 1".format(self.module_name))
+ _assert(self.R >= self.eta, "{0}: R should not be less than eta".format(self.module_name))
+ self.s_max = int(math.floor(math.log(self.R, self.eta)))
+ _assert(self.skip_last >= 0 and self.skip_last < self.s_max+1, "{0}: skip_last must be " \
+ "non-negative and less than {1}".format(self.module_name, self.s_max))
+
def find_hyperband_config(self):
"""
- Runs the diagonal hyperband algorithm.
+ Executes the diagonal hyperband algorithm.
"""
initial_vals = {}
@@ -308,6 +451,7 @@ class KerasAutoML():
r = self.R * math.pow(self.eta, -s) # initial number of iterations to run configurations for
initial_vals[s] = (n, int(round(r)))
self.start_training_time = self.get_current_timestamp()
+ self.start_training_time = get_current_timestamp(AutoMLConstants.TIME_FORMAT)
random_search = MstSearch(self.schema_madlib,
self.model_arch_table,
self.model_selection_table,
@@ -322,7 +466,7 @@ class KerasAutoML():
# for creating the summary table for usage in fit multiple
plpy.execute("CREATE TABLE {AutoMLSchema.TEMP_MST_SUMMARY_TABLE} AS " \
- "SELECT * FROM {random_search.model_selection_summary_table}".format(AutoMLSchema=AutoMLSchema,
+ "SELECT * FROM {random_search.model_selection_summary_table}".format(AutoMLSchema=AutoMLConstants,
random_search=random_search))
ranges_dict = self.mst_key_ranges_dict(initial_vals)
# to store the bracket and round numbers
@@ -347,10 +491,11 @@ class KerasAutoML():
num_iterations))
self.reconstruct_temp_mst_table(i, ranges_dict, configs_prune_lookup) # has keys to evaluate
- active_keys = plpy.execute("SELECT mst_key FROM {AutoMLSchema.TEMP_MST_TABLE}".format(AutoMLSchema=
- AutoMLSchema))
+ active_keys = plpy.execute("SELECT {ModelSelectionSchema.MST_KEY} " \
+ "FROM {AutoMLSchema.TEMP_MST_TABLE}".format(AutoMLSchema=AutoMLConstants,
+ ModelSelectionSchema=ModelSelectionSchema))
for k in active_keys:
- i_dict[k['mst_key']] += 1
+ i_dict[k[ModelSelectionSchema.MST_KEY]] += 1
self.warm_start = int(i != 0)
mcf = self.metrics_compute_frequency if self._is_valid_metrics_compute_frequency(num_iterations) else None
with SetGUC("plan_cache_mode", "force_generic_plan"):
@@ -359,15 +504,15 @@ class KerasAutoML():
self.validation_table, mcf, self.warm_start, self.name, self.description)
self.update_model_output_table(model_training)
self.update_model_output_info_table(i, model_training, initial_vals)
- self.end_training_time = self.get_current_timestamp()
+
+ self.print_best_mst_so_far()
+
+ self.end_training_time = get_current_timestamp(AutoMLConstants.TIME_FORMAT)
self.add_additional_info_cols(s_dict, i_dict)
self.update_model_selection_table()
self.generate_model_output_summary_table(model_training)
self.remove_temp_tables(model_training)
-
- def get_current_timestamp(self):
- """for start and end times for the chosen AutoML algorithm. Showcased in the output summary table"""
- return datetime.fromtimestamp(time()).strftime('%Y-%m-%d %H:%M:%S')
+ # cleanup_madlib_temp_tables(self.schema_madlib, AutoMLSchema.TARGET_SCHEMA)
def mst_key_ranges_dict(self, initial_vals):
"""
@@ -394,13 +539,15 @@ class KerasAutoML():
_assert_equal(len(configs_prune_lookup), 1, "invalid args")
lower_bound, upper_bound = ranges_dict[self.s_max]
plpy.execute("CREATE TABLE {AutoMLSchema.TEMP_MST_TABLE} AS SELECT * FROM {self.model_selection_table} "
- "WHERE mst_key >= {lower_bound} AND mst_key <= {upper_bound}".format(self=self,
- AutoMLSchema=AutoMLSchema,
- lower_bound=lower_bound,
- upper_bound=upper_bound,))
+ "WHERE {ModelSelectionSchema.MST_KEY} >= {lower_bound} " \
+ "AND {ModelSelectionSchema.MST_KEY} <= {upper_bound}".format(self=self,
+ AutoMLSchema=AutoMLConstants,
+ lower_bound=lower_bound,
+ upper_bound=upper_bound,
+ ModelSelectionSchema=ModelSelectionSchema))
return
# dropping and repopulating temp_mst_table
- drop_tables([AutoMLSchema.TEMP_MST_TABLE])
+ drop_tables([AutoMLConstants.TEMP_MST_TABLE])
# {mst_key} changed from SERIAL to INTEGER for safe insertions and preservation of mst_key values
create_query = """
@@ -411,29 +558,35 @@ class KerasAutoML():
{fit_params} VARCHAR,
unique ({model_id}, {compile_params}, {fit_params})
);
- """.format(AutoMLSchema=AutoMLSchema,
+ """.format(AutoMLSchema=AutoMLConstants,
mst_key=ModelSelectionSchema.MST_KEY,
model_id=ModelSelectionSchema.MODEL_ID,
compile_params=ModelSelectionSchema.COMPILE_PARAMS,
fit_params=ModelSelectionSchema.FIT_PARAMS)
- with MinWarning('warning'):
- plpy.execute(create_query)
+ # with MinWarning('warning'):
+ plpy.execute(create_query)
query = ""
new_configs = True
for s_val in configs_prune_lookup:
lower_bound, upper_bound = ranges_dict[s_val]
if new_configs:
- query += "INSERT INTO {AutoMLSchema.TEMP_MST_TABLE} SELECT mst_key, model_id, compile_params, fit_params " \
- "FROM {self.model_selection_table} WHERE mst_key >= {lower_bound} " \
- "AND mst_key <= {upper_bound};".format(self=self, AutoMLSchema=AutoMLSchema,
- lower_bound=lower_bound, upper_bound=upper_bound)
+ query += "INSERT INTO {AutoMLSchema.TEMP_MST_TABLE} SELECT {ModelSelectionSchema.MST_KEY}, " \
+ "{ModelSelectionSchema.MODEL_ID}, {ModelSelectionSchema.COMPILE_PARAMS}, " \
+ "{ModelSelectionSchema.FIT_PARAMS} FROM {self.model_selection_table} WHERE " \
+ "{ModelSelectionSchema.MST_KEY} >= {lower_bound} AND {ModelSelectionSchema.MST_KEY} <= " \
+ "{upper_bound};".format(self=self, AutoMLSchema=AutoMLConstants,
+ ModelSelectionSchema=ModelSelectionSchema,
+ lower_bound=lower_bound, upper_bound=upper_bound)
new_configs = False
else:
- query += "INSERT INTO {AutoMLSchema.TEMP_MST_TABLE} SELECT mst_key, model_id, compile_params, fit_params " \
- "FROM {self.model_info_table} WHERE mst_key >= {lower_bound} " \
- "AND mst_key <= {upper_bound} ORDER BY {AutoMLSchema.LOSS_METRIC} " \
- "LIMIT {configs_prune_lookup_val};".format(self=self, AutoMLSchema=AutoMLSchema,
+ query += "INSERT INTO {AutoMLSchema.TEMP_MST_TABLE} SELECT {ModelSelectionSchema.MST_KEY}, " \
+ "{ModelSelectionSchema.MODEL_ID}, {ModelSelectionSchema.COMPILE_PARAMS}, " \
+ "{ModelSelectionSchema.FIT_PARAMS} " \
+ "FROM {self.model_info_table} WHERE {ModelSelectionSchema.MST_KEY} >= {lower_bound} " \
+ "AND {ModelSelectionSchema.MST_KEY} <= {upper_bound} ORDER BY {AutoMLSchema.LOSS_METRIC} " \
+ "LIMIT {configs_prune_lookup_val};".format(self=self, AutoMLSchema=AutoMLConstants,
+ ModelSelectionSchema=ModelSelectionSchema,
lower_bound=lower_bound, upper_bound=upper_bound,
configs_prune_lookup_val=configs_prune_lookup[s_val])
plpy.execute(query)
@@ -459,8 +612,9 @@ class KerasAutoML():
# inserts any newly trained configs
plpy.execute("INSERT INTO {self.model_output_table} SELECT * FROM {model_training.original_model_output_table} " \
"WHERE {model_training.original_model_output_table}.mst_key NOT IN " \
- "(SELECT mst_key FROM {self.model_output_table})".format(self=self,
- model_training=model_training))
+ "(SELECT {ModelSelectionSchema.MST_KEY} FROM {self.model_output_table})".format(self=self,
+ model_training=model_training,
+ ModelSelectionSchema=ModelSelectionSchema))
def update_model_output_info_table(self, i, model_training, initial_vals):
"""
@@ -473,7 +627,7 @@ class KerasAutoML():
# normalizing factor for metrics_iters due to warm start
epochs_factor = sum([n[1] for n in initial_vals.values()][::-1][:i]) # i & initial_vals args needed
iters = plpy.execute("SELECT {AutoMLSchema.METRICS_ITERS} " \
- "FROM {model_training.model_summary_table}".format(AutoMLSchema=AutoMLSchema,
+ "FROM {model_training.model_summary_table}".format(AutoMLSchema=AutoMLConstants,
model_training=model_training))
metrics_iters_val = [epochs_factor+mi for mi in iters[0]['metrics_iters']] # global iteration counter
@@ -492,15 +646,16 @@ class KerasAutoML():
"training_loss=a.training_loss || t.training_loss, ".format(self=self) + validation_update_q +
"{AutoMLSchema.METRICS_ITERS}=a.metrics_iters || ARRAY{metrics_iters_val}::INTEGER[] " \
"FROM {model_training.model_info_table} t " \
- "WHERE a.mst_key=t.mst_key".format(model_training=model_training, AutoMLSchema=AutoMLSchema,
+ "WHERE a.mst_key=t.mst_key".format(model_training=model_training, AutoMLSchema=AutoMLConstants,
metrics_iters_val=metrics_iters_val))
# inserts info about metrics and validation for newly trained model configs
plpy.execute("INSERT INTO {self.model_info_table} SELECT t.*, ARRAY{metrics_iters_val}::INTEGER[] AS metrics_iters " \
"FROM {model_training.model_info_table} t WHERE t.mst_key NOT IN " \
- "(SELECT mst_key FROM {self.model_info_table})".format(self=self,
+ "(SELECT {ModelSelectionSchema.MST_KEY} FROM {self.model_info_table})".format(self=self,
model_training=model_training,
- metrics_iters_val=metrics_iters_val))
+ metrics_iters_val=metrics_iters_val,
+ ModelSelectionSchema=ModelSelectionSchema))
def add_additional_info_cols(self, s_dict, i_dict):
"""Adds s and i columns to the info table"""
@@ -512,63 +667,427 @@ class KerasAutoML():
"b (key integer, s_val integer, i_val integer) WHERE t.mst_key=b.key".format(self=self, l=l)
plpy.execute(query)
- def update_model_selection_table(self):
+# @MinWarning("warning")
+class AutoMLHyperopt(KerasAutoML):
+ """
+ This class implements Hyperopt, another automl method that explores awkward search spaces using
+ Random Search, Tree-structured Parzen Estimator (TPE), or Adaptive TPE.
+
+ This function executes hyperopt on top of our multiple model training infrastructure powered with
+ Model hOpper Parallelism (MOP), a hybrid of data and task parallelism.
+
+ This automl method inherits qualities from the automl class.
+ """
+ def __init__(self, schema_madlib, source_table, model_output_table, model_arch_table, model_selection_table,
+ model_id_list, compile_params_grid, fit_params_grid, automl_method,
+ automl_params, random_state=None, object_table=None,
+ use_gpus=False, validation_table=None, metrics_compute_frequency=None,
+ name=None, description=None, **kwargs):
+ automl_method = automl_method if automl_method else AutoMLConstants.HYPEROPT
+ automl_params = automl_params if automl_params else 'num_configs=20, num_iterations=5, algorithm=tpe'
+ KerasAutoML.__init__(self, schema_madlib, source_table, model_output_table, model_arch_table,
+ model_selection_table, model_id_list, compile_params_grid, fit_params_grid,
+ automl_method, automl_params, random_state, object_table, use_gpus,
+ validation_table, metrics_compute_frequency, name, description, **kwargs)
+ self.compile_params_grid = self.compile_params_grid.replace('\n', '').replace(' ', '')
+ self.fit_params_grid = self.fit_params_grid.replace('\n', '').replace(' ', '')
+ try:
+ self.compile_params_grid = literal_eval(self.compile_params_grid)
+
+ except:
+ plpy.error("Invalid syntax in 'compile_params_dict'")
+ try:
+ self.fit_params_grid = literal_eval(self.fit_params_grid)
+ except:
+ plpy.error("Invalid syntax in 'fit_params_dict'")
+ self.validate_and_define_inputs()
+ self.num_segments = self.get_num_segments()
+
+ self.create_model_output_table()
+ self.create_model_output_info_table()
+ self.find_hyperopt_config()
+
+ def get_num_segments(self):
"""
- Drops and re-create the mst table to only include the best performing model configuration.
+ # query dist rules from summary table to get the total no of segments
+ :return:
"""
- drop_tables([self.model_selection_table])
+ source_summary_table = add_postfix(self.source_table, '_summary')
+ dist_rules = plpy.execute("SELECT {0} from {1}".format(DISTRIBUTION_RULES, source_summary_table))[0][DISTRIBUTION_RULES]
+ #TODO create constant for all_segments
+ if dist_rules == "all_segments":
+ return get_seg_number()
- # only retaining best performing config
- plpy.execute("CREATE TABLE {self.model_selection_table} AS SELECT mst_key, model_id, compile_params, " \
- "fit_params FROM {self.model_info_table} " \
- "ORDER BY {AutoMLSchema.LOSS_METRIC} LIMIT 1".format(self=self, AutoMLSchema=AutoMLSchema))
+ return len(dist_rules)
- def generate_model_output_summary_table(self, model_training):
+ def validate_and_define_inputs(self):
+ automl_params_dict = extract_keyvalue_params(self.automl_params,
+ lower_case_names=True)
+ # casting relevant values to int
+ for i in automl_params_dict:
+ try:
+ automl_params_dict[i] = int(automl_params_dict[i])
+ except ValueError:
+ pass
+ _assert(len(automl_params_dict) >= 1 and len(automl_params_dict) <= 3,
+ "{0}: Only num_configs, num_iterations, and algorithm may be specified".format(self.module_name))
+ for i in automl_params_dict:
+ if i == AutoMLConstants.NUM_CONFIGS:
+ self.num_configs = automl_params_dict[AutoMLConstants.NUM_CONFIGS]
+ elif i == AutoMLConstants.NUM_ITERS:
+ self.num_iters = automl_params_dict[AutoMLConstants.NUM_ITERS]
+ elif i == AutoMLConstants.ALGORITHM:
+ if automl_params_dict[AutoMLConstants.ALGORITHM].lower() == 'rand':
+ self.algorithm = rand
+ elif automl_params_dict[AutoMLConstants.ALGORITHM].lower() == 'tpe':
+ self.algorithm = tpe
+ # elif automl_params_dict[AutoMLSchema.ALGORITHM].lower() == 'atpe':
+ # self.algorithm = atpe
+ # uncomment the above lines after atpe works # TODO
+ else:
+ plpy.error("{0}: valid algorithm 'automl_params' for hyperopt: 'rand', 'tpe'".format(self.module_name)) # , or 'atpe'
+ else:
+ plpy.error("{0}: {1} is an invalid automl param".format(self.module_name, i))
+ _assert(self.num_configs > 0 and self.num_iters > 0, "{0}: num_configs and num_iterations in 'automl_params' "
+ "must be > 0".format(self.module_name))
+ _assert(self._is_valid_metrics_compute_frequency(self.num_iters), "{0}: 'metrics_compute_frequency' "
+ "out of iteration range".format(self.module_name))
+
+ def find_hyperopt_config(self):
"""
- Creates and populates static values related to the AutoML workload.
- :param model_training: Fit Multiple function call object.
+ Executes hyperopt on top of MOP.
"""
- create_query = plpy.prepare("""
- CREATE TABLE {self.model_summary_table} AS
- SELECT
- $MAD${self.source_table}$MAD$::TEXT AS source_table,
- $MAD${self.validation_table}$MAD$::TEXT AS validation_table,
- $MAD${self.model_output_table}$MAD$::TEXT AS model,
- $MAD${self.model_info_table}$MAD$::TEXT AS model_info,
- (SELECT dependent_varname FROM {model_training.model_summary_table})
- AS dependent_varname,
- (SELECT independent_varname FROM {model_training.model_summary_table})
- AS independent_varname,
- $MAD${self.model_arch_table}$MAD$::TEXT AS model_arch_table,
- $MAD${self.model_selection_table}$MAD$::TEXT AS model_selection_table,
- $MAD${self.automl_method}$MAD$::TEXT AS automl_method,
- $MAD${self.automl_params}$MAD$::TEXT AS automl_params,
- $MAD${self.random_state}$MAD$::TEXT AS random_state,
- $MAD${self.object_table}$MAD$::TEXT AS object_table,
- {self.use_gpus} AS use_gpus,
- (SELECT metrics_compute_frequency FROM {model_training.model_summary_table})::INTEGER
- AS metrics_compute_frequency,
- $MAD${self.name}$MAD$::TEXT AS name,
- $MAD${self.description}$MAD$::TEXT AS description,
- '{self.start_training_time}'::TIMESTAMP AS start_training_time,
- '{self.end_training_time}'::TIMESTAMP AS end_training_time,
- (SELECT madlib_version FROM {model_training.model_summary_table}) AS madlib_version,
- (SELECT num_classes FROM {model_training.model_summary_table})::INTEGER AS num_classes,
- (SELECT class_values FROM {model_training.model_summary_table}) AS class_values,
- (SELECT dependent_vartype FROM {model_training.model_summary_table})
- AS dependent_vartype,
- (SELECT normalizing_const FROM {model_training.model_summary_table})
- AS normalizing_const
- """.format(self=self, model_training=model_training))
+ make_mst_summary = True
+ trials = Trials()
+ domain = Domain(None, self.get_search_space())
+ rand_state = np.random.RandomState(self.random_state)
+ configs_lst = self.get_configs_list(self.num_configs, self.num_segments)
- with MinWarning('warning'):
- plpy.execute(create_query)
+ self.start_training_time = get_current_timestamp(AutoMLConstants.TIME_FORMAT)
+ fit_multiple_runtime = 0
+ for low, high in configs_lst:
+ i, n = low, high - low + 1
- def remove_temp_tables(self, model_training):
+ # Using HyperOpt TPE/ATPE to generate parameters
+ hyperopt_params = []
+ sampled_params = []
+ for j in range(i, i + n):
+ new_param = self.algorithm.suggest([j], domain, trials, rand_state.randint(0, AutoMLConstants.INT_MAX))
+ new_param[0]['status'] = STATUS_RUNNING
+
+ trials.insert_trial_docs(new_param)
+ trials.refresh()
+ hyperopt_params.append(new_param[0])
+ sampled_params.append(new_param[0]['misc']['vals'])
+
+ model_id_list, compile_params, fit_params = self.extract_param_vals(sampled_params)
+ msts_list = self.generate_msts(model_id_list, compile_params, fit_params)
+ # cleanup_madlib_temp_tables(self.schema_madlib, AutoMLSchema.TARGET_SCHEMA)
+ try:
+ self.remove_temp_tables(model_training)
+ except:
+ pass
+ self.populate_temp_mst_tables(i, msts_list)
+
+ plpy.info("***Evaluating {n} newly suggested model configurations***".format(n=n))
+ fit_multiple_start_time = time.time()
+ model_training = FitMultipleModel(self.schema_madlib, self.source_table, AutoMLConstants.TEMP_OUTPUT_TABLE,
+ AutoMLConstants.TEMP_MST_TABLE, self.num_iters, self.use_gpus, self.validation_table,
+ self.metrics_compute_frequency, False, self.name, self.description, fit_multiple_runtime)
+ fit_multiple_runtime += time.time() - fit_multiple_start_time
+ if make_mst_summary:
+ self.generate_mst_summary_table(self.model_selection_summary_table)
+ make_mst_summary = False
+
+ # HyperOpt TPE update
+ for k, hyperopt_param in enumerate(hyperopt_params, i):
+ loss_val = plpy.execute("SELECT {AutoMLSchema.LOSS_METRIC} FROM {model_training.model_info_table} " \
+ "WHERE {ModelSelectionSchema.MST_KEY}={k}".format(AutoMLSchema=AutoMLConstants,
+ ModelSelectionSchema=ModelSelectionSchema,
+ **locals()))[0][AutoMLConstants.LOSS_METRIC]
+
+ # avoid removing the two lines below (part of Hyperopt updates)
+ hyperopt_param['status'] = STATUS_OK
+ hyperopt_param['result'] = {'loss': loss_val, 'status': STATUS_OK}
+ trials.refresh()
+
+ # stacks info of all model configs together
+ self.update_model_output_and_info_tables(model_training)
+
+ self.print_best_mst_so_far()
+
+ self.end_training_time = get_current_timestamp(AutoMLConstants.TIME_FORMAT)
+ self.update_model_selection_table()
+ self.generate_model_output_summary_table(model_training)
+ # cleanup_madlib_temp_tables(self.schema_madlib, AutoMLSchema.TARGET_SCHEMA)
+ self.remove_temp_tables(model_training)
+
+ def get_configs_list(self, num_configs, num_segments):
"""
- Remove all intermediate tables created for AutoML runs/updates.
- :param model_training: Fit Multiple function call object.
+ Gets schedule to evaluate model configs
+ :return: Model configs evaluation schedule
"""
- drop_tables([model_training.original_model_output_table, model_training.model_info_table,
- model_training.model_summary_table, AutoMLSchema.TEMP_MST_TABLE,
- AutoMLSchema.TEMP_MST_SUMMARY_TABLE])
+ num_buckets = int(round(float(num_configs) / num_segments))
+ configs_list = []
+ start_idx = 1
+ models_populated = 0
+ for _ in range(num_buckets - 1):
+ end_idx = start_idx + num_segments
+ models_populated += num_segments
+ configs_list.append((start_idx, end_idx - 1))
+ start_idx = end_idx
+
+ remaining_models = num_configs - models_populated
+ configs_list.append((start_idx, start_idx + remaining_models-1))
+
+ return configs_list
+
+ def get_search_space(self):
+ """
+ Converts user inputs to hyperopt search space.
+ :return: Hyperopt search space
+ """
+
+ # initial params (outside 'optimizer_params_list')
+ hyperopt_search_dict = {}
+ hyperopt_search_dict['model_id'] = self.get_hyperopt_exps('model_id', self.model_id_list)
+
+
+ for j in self.fit_params_grid:
+ hyperopt_search_dict[j] = self.get_hyperopt_exps(j, self.fit_params_grid[j])
+
+ for i in self.compile_params_grid:
+ if i != ModelSelectionSchema.OPTIMIZER_PARAMS_LIST:
+ hyperopt_search_dict[i] = self.get_hyperopt_exps(i, self.compile_params_grid[i])
+
+ hyperopt_search_space_lst = []
+
+ counter = 1 # for unique names to allow multiple distribution options for optimizer params
+ for optimizer_dict in self.compile_params_grid[ModelSelectionSchema.OPTIMIZER_PARAMS_LIST]:
+ for o_param in optimizer_dict:
+ name = o_param + '_' + str(counter)
+ hyperopt_search_dict[name] = self.get_hyperopt_exps(name, optimizer_dict[o_param])
+ # appending deep copy
+ hyperopt_search_space_lst.append({k:v for k, v in hyperopt_search_dict.items()})
+ for o_param in optimizer_dict:
+ name = o_param + '_' + str(counter)
+ del hyperopt_search_dict[name]
+ counter += 1
+
+ return hp.choice('space', hyperopt_search_space_lst)
+
+ def get_hyperopt_exps(self, cp, param_value_list):
+ """
+ Samples a value from a given list of values, either randomly from a list of discrete elements,
+ or from a specified distribution.
+ :param cp: compile param
+ :param param_value_list: list of values (or specified distribution) for a param
+ :return: sampled value
+ """
+ # check if need to sample from a distribution
+ if type(param_value_list[-1]) == str and all([type(i) != str and not callable(i) for i in param_value_list[:-1]]) \
+ and len(param_value_list) > 1:
+ _assert_equal(len(param_value_list), 3,
+ "{0}: '{1}' should have exactly 3 elements if picking from a distribution".format(self.module_name, cp))
+ _assert(param_value_list[1] > param_value_list[0],
+ "{0}: '{1}' should be of the format [lower_bound, upper_bound, distribution_type]".format(self.module_name, cp))
+ if param_value_list[-1] == 'linear':
+ return hp.uniform(cp, param_value_list[0], param_value_list[1])
+ elif param_value_list[-1] == 'log':
+ return hp.loguniform(cp, np.log(param_value_list[0]), np.log(param_value_list[1]))
+ else:
+ plpy.error("{0}: Please choose a valid distribution type for '{1}': {2}".format(
+ self.module_name,
+ self.original_param_details(cp)[0],
+ ['linear', 'log']))
+ else:
+ # random sampling
+ return hp.choice(cp, param_value_list)
+
+ def extract_param_vals(self, sampled_params):
+ """
+ Extract parameter values from hyperopt search space.
+ :param sampled_params: params suggested by hyperopt.
+ :return: lists of model ids, compile and fit params.
+ """
+ model_id_list, compile_params, fit_params = [], [], []
+ for params_dict in sampled_params:
+ compile_dict, fit_dict, optimizer_params_dict = {}, {}, {}
+ for p in params_dict:
+ if len(params_dict[p]) == 0 or p == 'space':
+ continue
+ val = params_dict[p][0]
+ if p == 'model_id':
+ model_id_list.append(self.model_id_list[val])
+ continue
+ elif p in self.fit_params_grid:
+ try:
+ # check if params_dict[p] is an index
+ fit_dict[p] = self.fit_params_grid[p][val]
+ except TypeError:
+ fit_dict[p] = params_dict[p]
+ elif p in self.compile_params_grid:
+ try:
+ # check if params_dict[p] is an index
+ compile_dict[p] = self.compile_params_grid[p][val]
+ except TypeError:
+ compile_dict[p] = val
+ else:
+ o_param, idx = self.original_param_details(p) # extracting unique attribute
+ try:
+ # check if params_dict[p] is an index (i.e. optimizer, for example)
+ optimizer_params_dict[o_param] = self.compile_params_grid[
+ ModelSelectionSchema.OPTIMIZER_PARAMS_LIST][idx][o_param][val]
+ except TypeError:
+ optimizer_params_dict[o_param] = val
+ compile_dict[ModelSelectionSchema.OPTIMIZER_PARAMS_LIST] = optimizer_params_dict
+
+ compile_params.append(compile_dict)
+ fit_params.append(fit_dict)
+
+ return model_id_list, compile_params, fit_params
+
+ def original_param_details(self, name):
+ """
+ Returns the original param name and book-keeping detail.
+ :param name: name of the param (example - lr_1, epsilon_12)
+ :return: original param name and book-keeping position.
+ """
+ parts = name.split('_')
+ return '_'.join(parts[:-1]), int(parts[-1]) - 1
+
+
+ def generate_msts(self, model_id_list, compile_params, fit_params):
+ """
+ Generates msts to insert in the mst table.
+ :param model_id_list: list of model ids
+ :param compile_params: list compile params
+ :param fit_params:list of fit params
+ :return: List of msts to insert in the mst table.
+ """
+ assert len(model_id_list) == len(compile_params) == len(fit_params)
+ msts = []
+
+ for i in range(len(compile_params)):
+ combination = {}
+ combination[ModelSelectionSchema.MODEL_ID] = model_id_list[i]
+ combination[ModelSelectionSchema.COMPILE_PARAMS] = generate_row_string(compile_params[i])
+ combination[ModelSelectionSchema.FIT_PARAMS] = generate_row_string(fit_params[i])
+ msts.append(combination)
+
+ return msts
+
+ def populate_temp_mst_tables(self, i, msts_list):
+ """
+ Creates and populates temp mst and summary tables with newly suggested model configs for evaluation.
+ :param i: mst key number
+ :param msts_list: list of generated msts.
+ """
+ # extra sanity check
+ if table_exists(AutoMLConstants.TEMP_MST_TABLE):
+ drop_tables([AutoMLConstants.TEMP_MST_TABLE])
+
+ create_query = """
+ CREATE TABLE {AutoMLSchema.TEMP_MST_TABLE} (
+ {mst_key} INTEGER,
+ {model_id} INTEGER,
+ {compile_params} VARCHAR,
+ {fit_params} VARCHAR,
+ unique ({model_id}, {compile_params}, {fit_params})
+ );
+ """.format(AutoMLSchema=AutoMLConstants,
+ mst_key=ModelSelectionSchema.MST_KEY,
+ model_id=ModelSelectionSchema.MODEL_ID,
+ compile_params=ModelSelectionSchema.COMPILE_PARAMS,
+ fit_params=ModelSelectionSchema.FIT_PARAMS)
+ # with MinWarning('warning'):
+ plpy.execute(create_query)
+ mst_key_val = i
+ for mst in msts_list:
+ model_id = mst[ModelSelectionSchema.MODEL_ID]
+ compile_params = mst[ModelSelectionSchema.COMPILE_PARAMS]
+ fit_params = mst[ModelSelectionSchema.FIT_PARAMS]
+ insert_query = """
+ INSERT INTO
+ {AutoMLSchema.TEMP_MST_TABLE}(
+ {mst_key_col},
+ {model_id_col},
+ {compile_params_col},
+ {fit_params_col}
+ )
+ VALUES (
+ {mst_key_val},
+ {model_id},
+ $${compile_params}$$,
+ $${fit_params}$$
+ )
+ """.format(mst_key_col=ModelSelectionSchema.MST_KEY,
+ model_id_col=ModelSelectionSchema.MODEL_ID,
+ compile_params_col=ModelSelectionSchema.COMPILE_PARAMS,
+ fit_params_col=ModelSelectionSchema.FIT_PARAMS,
+ AutoMLSchema=AutoMLConstants,
+ **locals())
+ mst_key_val += 1
+ plpy.execute(insert_query)
+
+ self.generate_mst_summary_table(AutoMLConstants.TEMP_MST_SUMMARY_TABLE)
+
+ def generate_mst_summary_table(self, tbl_name):
+ """
+ generates mst summary table with the given name
+ :param tbl_name: name of summary table
+ """
+ _assert(tbl_name.endswith('_summary'), 'invalid summary table name')
+
+ # extra sanity check
+ if table_exists(tbl_name):
+ drop_tables([tbl_name])
+
+ create_query = """
+ CREATE TABLE {tbl_name} (
+ {model_arch_table} VARCHAR,
+ {object_table} VARCHAR
+ );
+ """.format(tbl_name=tbl_name,
+ model_arch_table=ModelSelectionSchema.MODEL_ARCH_TABLE,
+ object_table=ModelSelectionSchema.OBJECT_TABLE)
+ # with MinWarning('warning'):
+ plpy.execute(create_query)
+
+ if self.object_table is None:
+ object_table = 'NULL::VARCHAR'
+ else:
+ object_table = '$${0}$$'.format(self.object_table)
+ insert_summary_query = """
+ INSERT INTO
+ {tbl_name}(
+ {model_arch_table_name},
+ {object_table_name}
+ )
+ VALUES (
+ $${self.model_arch_table}$$,
+ {object_table}
+ )
+ """.format(model_arch_table_name=ModelSelectionSchema.MODEL_ARCH_TABLE,
+ object_table_name=ModelSelectionSchema.OBJECT_TABLE,
+ **locals())
+ plpy.execute(insert_summary_query)
+
+ def update_model_output_and_info_tables(self, model_training):
+ """
+ Updates model output and info tables by stacking rows after each evaluation round.
+ :param model_training: Fit Multiple class object
+ """
+ metrics_iters = plpy.execute("SELECT {AutoMLSchema.METRICS_ITERS} " \
+ "FROM {model_training.original_model_output_table}_summary".format(self=self,
+ model_training=model_training,
+ AutoMLSchema=AutoMLConstants))[0][AutoMLConstants.METRICS_ITERS]
+ if metrics_iters:
+ metrics_iters = "ARRAY{0}".format(metrics_iters)
+ # stacking new rows from training
+ plpy.execute("INSERT INTO {self.model_output_table} SELECT * FROM " \
+ "{model_training.original_model_output_table}".format(self=self, model_training=model_training))
+ plpy.execute("INSERT INTO {self.model_info_table} SELECT *, {metrics_iters} FROM " \
+ "{model_training.model_info_table}".format(self=self,
+ model_training=model_training,
+ metrics_iters=metrics_iters))
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in
index 06889d2..98617d7 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in
@@ -169,20 +169,25 @@ madlib_keras_automl(
<dt>automl_method (optional)</dt>
<dd>VARCHAR, default 'hyperband'. Name of the automl algorithm to run.
- Currently only support hyperband. Note that you can also use short prefixes
- for the 'hyperband' keyword, e.g.,'hyper' or 'hyp' instead
- of writing out 'hyperband' in full.
+ Can be either 'hyperband' or hyperopt'. Prefixing is not supported but arg value can be case insensitive.
</dd>
<dt>automl_params (optional)</dt>
- <dd>VARCHAR, default 'R=6, eta=3, skip_last=0'. Parameters for the chosen automl
- method in a comma-separated string of key-value pairs. Hyperband params are:
+ <dd>VARCHAR, default 'R=6, eta=3, skip_last=0' (for Hyperband). Parameters for the chosen automl method in a
+ comma-separated string of key-value pairs. For eg - 'num_configs=20, num_iterations=5, algorithm=tpe' for Hyperopt
+ Hyperband params are:
R - the maximum amount of resources/iterations allocated to a single configuration
- in a round of hyperband, eta - factor controlling the proportion of configurations discarded in each
- round of successive halving, skip_last - number of last diagonal brackets to skip running
+ in a round of hyperband,
+ eta - factor controlling the proportion of configurations discarded in each
+ round of successive halving,
+ skip_last - number of last diagonal brackets to skip running
in the algorithm.
We encourage setting an low R value (i.e. 2 to 10), or a high R value and a high skip_last value to evaluate
a variety of configurations with decent number of iterations. See the description below for details.
+ Hyperopt params are:
+ num_configs - total number of model configurations to evaluate,
+ num_iterations - fixed number of iterations for evaluating each model configurations,
+ algorithm - name of algorithm to explore search space in hyperopt ('rand', 'tpe', 'atpe').
</dd>
<dt>random_state (optional)</dt>
@@ -627,7 +632,6 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.hyperband_schedule(
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
-
CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_automl(
source_table VARCHAR,
model_output_table VARCHAR,
@@ -637,7 +641,7 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_automl(
compile_params_grid VARCHAR,
fit_params_grid VARCHAR,
automl_method VARCHAR DEFAULT 'hyperband',
- automl_params VARCHAR DEFAULT 'R=6, eta=3, skip_last=0',
+ automl_params VARCHAR DEFAULT NULL,
random_state INTEGER DEFAULT NULL,
object_table VARCHAR DEFAULT NULL,
use_gpus BOOLEAN DEFAULT FALSE,
@@ -648,6 +652,11 @@ CREATE OR REPLACE FUNCTION MADLIB_SCHEMA.madlib_keras_automl(
) RETURNS VOID AS $$
PythonFunctionBodyOnly(`deep_learning', `madlib_keras_automl')
with AOControl(False):
- schedule_loader = madlib_keras_automl.KerasAutoML(**globals())
+ if automl_method is None or automl_method.lower() == 'hyperband':
+ schedule_loader = madlib_keras_automl.AutoMLHyperband(**globals())
+ elif automl_method.lower() == 'hyperopt':
+ schedule_loader = madlib_keras_automl.AutoMLHyperopt(**globals())
+ else:
+ plpy.error("madlib_keras_automl: The chosen automl method must be 'hyperband' or 'hyperopt'")
$$ LANGUAGE plpythonu VOLATILE
m4_ifdef(`__HAS_FUNCTION_PROPERTIES__', `MODIFIES SQL DATA', `');
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
index c821474..1e49261 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_fit_multiple_model.py_in
@@ -81,7 +81,26 @@ class FitMultipleModel():
model_selection_table, num_iterations,
use_gpus=False, validation_table=None,
metrics_compute_frequency=None, warm_start=False, name="",
- description="", use_caching=False, **kwargs):
+ description="", use_caching=False, metrics_elapsed_time_offset=0, **kwargs):
+ """
+
+ :param schema_madlib: schema name
+ :param source_table: input table containing training dataset
+ :param model_output_table: output table
+ :param model_selection_table: input table containing model configs
+ :param num_iterations: number of iterations to train
+ :param use_gpus: determines whether GPUs are to be used for training
+ :param validation_table: input table containing the validation dataset
+ :param metrics_compute_frequency: Frequency to compute per-iteration metrics
+ for the training dataset and validation dataset
+ :param warm_start: indicates whether to initialize weights with the coefficients
+ from the last call of the fit function
+ :param name: name
+ :param description: description
+ :param metrics_elapsed_time_offset: time elapsed for the previous call to fit_multiple
+ (internal param used by automl to accumulate
+ metrics_elapsed_time)
+ """
# set the random seed for visit order/scheduling
random.seed(1)
if is_platform_pg():
@@ -117,6 +136,7 @@ class FitMultipleModel():
self.use_gpus = use_gpus
self.segments_per_host = get_segments_per_host()
self.cached_source_table = unique_string('cached_source_table')
+ self.metrics_elapsed_time_offset = metrics_elapsed_time_offset
if self.use_gpus:
self.accessible_gpus_for_seg = get_accessible_gpus_for_seg(
self.schema_madlib, self.segments_per_host, self.module_name)
@@ -283,7 +303,7 @@ class FitMultipleModel():
self.model_output_table,
mst[self.mst_key_col])
mst_metric_eval_time[mst[self.mst_key_col]] \
- .append(time.time() - self.metrics_elapsed_start_time)
+ .append(self.metrics_elapsed_time_offset + (time.time() - self.metrics_elapsed_start_time))
mst_loss[mst[self.mst_key_col]].append(loss)
mst_metric[mst[self.mst_key_col]].append(metric)
self.info_str += "\n\tmst_key={0}: metric={1}, loss={2}".format(mst[self.mst_key_col], metric, loss)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
index ca54a4d..96c2817 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_helper.py_in
@@ -26,6 +26,7 @@ from utilities.validate_args import table_exists
from madlib_keras_gpu_info import GPUInfoFunctions
import plpy
from math import isnan
+# from madlib_keras_model_selection import ModelSelectionSchema
############### Constants used in other deep learning files #########
# Name of columns in model summary table.
@@ -53,7 +54,7 @@ SMALLINT_SQL_TYPE = 'SMALLINT'
DEFAULT_NORMALIZING_CONST = 1.0
GP_SEGMENT_ID_COLNAME = "gp_segment_id"
INTERNAL_GPU_CONFIG = '__internal_gpu_config__'
-
+DISTRIBUTION_RULES = "distribution_rules"
#####################################################################
# Prepend a dimension to np arrays using expand_dims.
@@ -353,3 +354,57 @@ def get_metrics_sql_string(metrics_list, is_metrics_specified=True):
metrics_final = metrics_all = 'NULL'
return metrics_final, metrics_all
+
+def generate_row_string(configs_dict):
+ """
+ Generate row strings for MST table.
+ :param configs_dict: Dictionary of params configs (preferably either only compile params
+ or only fit params).
+ :return: string to insert as a row value in MST table.
+ """
+ result_row_string = ""
+ opl = 'optimizer_params_list'
+
+ if opl in configs_dict:
+ optimizer_params_dict = configs_dict[opl]
+ if 'optimizer' in optimizer_params_dict:
+ if optimizer_params_dict['optimizer'].lower() == 'sgd':
+ optimizer_value = "SGD"
+ elif optimizer_params_dict['optimizer'].lower() == 'rmsprop':
+ optimizer_value = "RMSprop"
+ else:
+ optimizer_value = optimizer_params_dict['optimizer'].capitalize()
+ opt_string = "optimizer" + "=" + "'" + str(optimizer_value) \
+ + "()" + "'"
+ else:
+ opt_string = "optimizer='RMSprop()'" # default optimizer
+ opt_param_string = ""
+ for opt_param in optimizer_params_dict:
+ if opt_param == 'optimizer':
+ continue
+ opt_param_string += opt_param + '=' + str(optimizer_params_dict[opt_param]) + ','
+ if opt_param_string == "":
+ result_row_string += opt_string
+ else:
+ opt_param_string = opt_param_string[:-1] # to exclude the last comma
+ part = opt_string.split('(')
+ result_row_string += part[0] + '(' + opt_param_string + part[1]
+
+ for c in configs_dict:
+ if c == opl:
+ continue
+ elif c == 'metrics':
+ if callable(configs_dict[c]):
+ result_row_string += "," + str(c) + "=" + "[" + str(configs_dict[c]) + "]"
+ else:
+ result_row_string += "," + str(c) + "=" + "['" + str(configs_dict[c]) + "']"
+ else:
+ if type(configs_dict[c]) == str or type(configs_dict[c]) == np.string_:
+ result_row_string += "," + str(c) + "=" + "'" + str(configs_dict[c]) + "'"
+ else:
+ # ints, floats, none type, booleans
+ result_row_string += "," + str(c) + "=" + str(configs_dict[c])
+
+ if result_row_string[0] == ',':
+ return result_row_string[1:]
+ return result_row_string
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
index 99c7150..3ea37dc 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_model_selection.py_in
@@ -27,12 +27,13 @@ import plpy
from copy import deepcopy
from madlib_keras_custom_function import CustomFunctionSchema
+from madlib_keras_helper import generate_row_string
from madlib_keras_validator import MstLoaderInputValidator
from madlib_keras_wrapper import convert_string_of_args_to_dict
from madlib_keras_wrapper import parse_and_validate_fit_params
from madlib_keras_wrapper import parse_and_validate_compile_params
from utilities.control import MinWarning
-from utilities.utilities import add_postfix, extract_keyvalue_params, _assert, _assert_equal
+from utilities.utilities import add_postfix, _assert, _assert_equal, extract_keyvalue_params
from utilities.utilities import quote_ident, get_schema
from utilities.validate_args import table_exists, drop_tables
@@ -462,8 +463,8 @@ class MstSearch():
fit_configs[k] = config[k]
else:
plpy.error("DL: {0} is an unidentified key".format(k))
- combination[ModelSelectionSchema.COMPILE_PARAMS] = self.generate_row_string(compile_configs)
- combination[ModelSelectionSchema.FIT_PARAMS] = self.generate_row_string(fit_configs)
+ combination[ModelSelectionSchema.COMPILE_PARAMS] = generate_row_string(compile_configs)
+ combination[ModelSelectionSchema.FIT_PARAMS] = generate_row_string(fit_configs)
self.msts.append(combination)
def find_random_combinations(self):
@@ -479,9 +480,9 @@ class MstSearch():
seed_changes += 1
combination[ModelSelectionSchema.MODEL_ID] = np.random.choice(self.model_id_list)
compile_dict, seed_changes = self.generate_param_config(self.compile_params_dict, seed_changes)
- combination[ModelSelectionSchema.COMPILE_PARAMS] = self.generate_row_string(compile_dict)
+ combination[ModelSelectionSchema.COMPILE_PARAMS] = generate_row_string(compile_dict)
fit_dict, seed_changes = self.generate_param_config(self.fit_params_dict, seed_changes)
- combination[ModelSelectionSchema.FIT_PARAMS] = self.generate_row_string(fit_dict)
+ combination[ModelSelectionSchema.FIT_PARAMS] = generate_row_string(fit_dict)
self.msts.append(combination)
def generate_param_config(self, params_dict, seed_changes):
@@ -542,58 +543,6 @@ class MstSearch():
# random sampling
return np.random.choice(param_value_list)
- def generate_row_string(self, configs_dict):
- """
- Generate row strings for MST table.
- :param configs_dict: Dictionary of params config.
- :return: string to insert as a row in MST table.
- """
- result_row_string = ""
-
- if ModelSelectionSchema.OPTIMIZER_PARAMS_LIST in configs_dict:
- optimizer_params_dict = configs_dict[ModelSelectionSchema.OPTIMIZER_PARAMS_LIST]
- if 'optimizer' in optimizer_params_dict:
- if optimizer_params_dict['optimizer'].lower() == 'sgd':
- optimizer_value = "SGD"
- elif optimizer_params_dict['optimizer'].lower() == 'rmsprop':
- optimizer_value = "RMSprop"
- else:
- optimizer_value = optimizer_params_dict['optimizer'].capitalize()
- opt_string = "optimizer" + "=" + "'" + str(optimizer_value) \
- + "()" + "'"
- else:
- opt_string = "optimizer='RMSprop()'" # default optimizer
- opt_param_string = ""
- for opt_param in optimizer_params_dict:
- if opt_param == 'optimizer':
- continue
- opt_param_string += opt_param + '=' + str(optimizer_params_dict[opt_param]) + ','
- if opt_param_string == "":
- result_row_string += opt_string
- else:
- opt_param_string = opt_param_string[:-1] # to exclude the last comma
- part = opt_string.split('(')
- result_row_string += part[0] + '(' + opt_param_string + part[1]
-
- for c in configs_dict:
- if c == ModelSelectionSchema.OPTIMIZER_PARAMS_LIST:
- continue
- elif c == 'metrics':
- if callable(configs_dict[c]):
- result_row_string += "," + str(c) + "=" + "[" + str(configs_dict[c]) + "]"
- else:
- result_row_string += "," + str(c) + "=" + "['" + str(configs_dict[c]) + "']"
- else:
- if type(configs_dict[c]) == str or type(configs_dict[c]) == np.string_:
- result_row_string += "," + str(c) + "=" + "'" + str(configs_dict[c]) + "'"
- else:
- # ints, floats, none type, booleans
- result_row_string += "," + str(c) + "=" + str(configs_dict[c])
-
- if result_row_string[0] == ',':
- return result_row_string[1:]
- return result_row_string
-
def create_mst_table(self):
"""Initialize the output mst table, if it doesn't exist (for incremental loading).
"""
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
index 9382407..8b2157d 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_validator.py_in
@@ -248,7 +248,7 @@ class InputValidator:
gpu_config = plpy.execute(
"SELECT {0} FROM {1}".format(INTERNAL_GPU_CONFIG, summary_table)
)[0][INTERNAL_GPU_CONFIG]
- if gpu_config == 'all_segments':
+ if gpu_config == DistributionRulesOptions.ALL_SEGMENTS:
_assert(0 not in accessible_gpus_for_seg,
"{0} error: Host(s) are missing gpus.".format(module_name))
else:
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
index 0516687..da9fb8a 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
@@ -29,21 +29,82 @@ m4_include(`SQLCommon.m4')
m4_changequote(`<!', `!>')
m4_ifdef(<!__POSTGRESQL__!>, <!!>, <!
---------------------------- MADLIB KERAS AUTOML HYPERBAND TEST CASES ---------------------------
-
--- test table dimensions / happy path
+--------------------------- HYPEROPT TEST CASES ---------------------------
+-- test table dimensions / happy path (algorithm = rand)
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
+automl_mst_table_summary;
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$);
+ ARRAY[1], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [{'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.01, 0.011, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [50], 'epochs': [1]}$$,
+ 'hyperopt', 'num_configs=5, num_iterations=6, algorithm=rand', NULL, NULL, FALSE, NULL, 1, 'test1', 'test1 descr');
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output;
SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output_info;
+-- Validate model output summary table
+SELECT assert(
+ source_table = 'iris_data_packed' AND
+ validation_table IS NULL AND
+ model = 'automl_output' AND
+ model_info = 'automl_output_info' AND
+ dependent_varname = 'class_text' AND
+ independent_varname = 'attributes' AND
+ model_arch_table = 'iris_model_arch' AND
+ model_selection_table = 'automl_mst_table' AND
+ automl_method = 'hyperopt' AND
+ automl_params = 'num_configs=5, num_iterations=6, algorithm=rand' AND
+ random_state IS NULL AND
+ object_table IS NULL AND
+ use_gpus = FALSE AND
+ metrics_compute_frequency = 1 AND
+ name = 'test1' AND
+ description = 'test1 descr' AND
+ start_training_time < now() AND
+ end_training_time < now() AND
+ madlib_version IS NOT NULL AND
+ num_classes = 3 AND
+ class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype = 'character varying' AND
+ normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
+) FROM (SELECT * FROM automl_output_summary) summary;
+
+-- Validate output info table for metrics_iters NOT NULL
+SELECT assert(
+ metrics_iters = ARRAY[1,2,3,4,5,6], 'Invalid metrics_iters value in output info table. Actual:' || __to_char(info)
+) FROM (SELECT * FROM automl_output_info) info;
+
+-- Validate mst summary table
+SELECT assert(
+ model_arch_table = 'iris_model_arch' AND
+ object_table IS NULL , 'mst summary table validation failed. Actual:' || __to_char(summary)
+) FROM (SELECT * FROM automl_mst_table_summary) summary;
+
+-- Validating the best model selected learns (training loss goes down and accuracy improves)
+-- TODO: Keep a look for flaky test
+SELECT assert(
+ training_loss[6]-training_loss[1] < 10e-4 AND
+ training_metrics[6]-training_metrics[1] > 0,
+ 'The loss and accuracy should have improved with more iterations.'
+)
+FROM automl_output_info
+WHERE mst_key = (SELECT mst_key from automl_mst_table);
+
+-- algorithm = tpe
+DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
+automl_mst_table_summary;
+SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperopt', 'num_configs=4, num_iterations=1, algorithm=tpe');
+
+SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
+SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
+SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
+SELECT assert(COUNT(*)=4, 'The length of table does not match with the inputs') FROM automl_output;
+SELECT assert(COUNT(*)=4, 'The length of table does not match with the inputs') FROM automl_output_info;
-- test invalid source table
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
@@ -51,241 +112,345 @@ DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, a
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('invalid_source_table', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid source table');
-- test preexisting output table
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+CREATE TABLE automl_output(a int);
DROP TABLE IF EXISTS automl_mst_table, automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for preexisting output table');
-- test preexisting selection table
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
-
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary;
+CREATE TABLE automl_mst_table(a int);
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for preexisting selection table');
--- test test invalid model id
+-- test invalid model id
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[2,-1], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid model id');
--- test invalid automl method
+-- test invalid distribution
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperbrand', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
-$TRAP$)=1, 'Should error out for invalid automl method');
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for preexisting selection table');
+-- test invalid automl method
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hb', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyper', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid automl method');
--- test invalid automl params {R, eta, skip_last}
+-- test invalid automl params for hyperopt: {num_configs, num_iterations, algorithm}
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=2, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
-$TRAP$)=1, 'Should error out for invalid automl params');
-
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
-SELECT assert(trap_error($TRAP$
- SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=0, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=-2, num_iterations=5, algorithm=rand', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid automl params');
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=1, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=2, num_iterations=0, algorithm=tpe', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid automl params');
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=3', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=random', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid automl params');
-- test invalid object table
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, 'invalid_object_table', FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, 'invalid_object_table', FALSE, NULL, NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid object table');
-- test invalid validation table
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
SELECT assert(trap_error($TRAP$
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, 'invalid_validation_table', NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=tpe', NULL, NULL, FALSE, 'invalid_validation_table', NULL, NULL, NULL);
$TRAP$)=1, 'Should error out for invalid validation table');
--- test automl_method val
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+-- test config reproducibility
+DROP TABLE IF EXISTS automl_output1, automl_output1_info, automl_output1_summary, automl_mst_table1,
+ automl_mst_table1_summary;
+SELECT madlib_keras_automl('iris_data_packed', 'automl_output1', 'iris_model_arch', 'automl_mst_table1',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyper', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperopt', 'num_configs=4, num_iterations=2, algorithm=tpe', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=15, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=15, 'The length of table does not match with the inputs') FROM automl_output_info;
+DROP TABLE IF EXISTS automl_output2, automl_output2_info, automl_output2_summary, automl_mst_table2,
+ automl_mst_table2_summary;
+SELECT madlib_keras_automl('iris_data_packed', 'automl_output2', 'iris_model_arch', 'automl_mst_table2',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperopt', 'num_configs=4, num_iterations=2, algorithm=tpe', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
+
+DROP TABLE IF EXISTS automl_output3, automl_output3_info, automl_output3_summary, automl_mst_table3,
+ automl_mst_table3_summary;
+SELECT madlib_keras_automl('iris_data_packed', 'automl_output3', 'iris_model_arch', 'automl_mst_table3',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'linear']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperopt', 'num_configs=4, num_iterations=2, algorithm=tpe', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
+
+SELECT assert(model_id=(SELECT model_id FROM automl_output2_info WHERE mst_key=3) AND
+ compile_params=(SELECT compile_params FROM automl_output2_info WHERE mst_key=3) AND
+ fit_params=(SELECT fit_params FROM automl_output2_info WHERE mst_key=3), 'invalid config uniformity')
+FROM (SELECT model_id, compile_params, fit_params FROM automl_output1_info WHERE mst_key=3) output1;
+SELECT assert(model_id=(SELECT model_id FROM automl_output2_info WHERE mst_key=3) AND
+ compile_params=(SELECT compile_params FROM automl_output2_info WHERE mst_key=3) AND
+ fit_params=(SELECT fit_params FROM automl_output2_info WHERE mst_key=3), 'invalid config uniformity')
+FROM (SELECT model_id, compile_params, fit_params FROM automl_output3_info WHERE mst_key=3) output3;
+
+-- Test for metrics_elapsed_time for 2 configs per trial (total 3 trials)
+-- Setup for distributing data only on 2 segments
+DROP TABLE IF EXISTS segments_to_use;
+CREATE TABLE segments_to_use(
+ dbid INTEGER,
+ hostname TEXT
+);
+INSERT INTO segments_to_use SELECT dbid, hostname
+ FROM gp_segment_configuration
+ WHERE content>=0 AND preferred_role='p' limit 2;
+
+DROP TABLE IF EXISTS iris_data_2seg_packed, iris_data_2seg_packed_summary;
+SELECT training_preprocessor_dl('iris_data', -- Source table
+ 'iris_data_2seg_packed', -- Output table
+ 'class_text', -- Dependent variable
+ 'attributes', -- Independent variable
+ NULL, 255, NULL,
+ 'segments_to_use' -- Distribution rules
+ );
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
+automl_mst_table_summary;
+SELECT madlib_keras_automl('iris_data_2seg_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [{'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.01, 0.011, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [50], 'epochs': [1]}$$,
+ 'hyperopt', 'num_configs=5, num_iterations=2, algorithm=rand', NULL, NULL, FALSE, NULL, 1);
+
+SELECT assert(
+ t1.metrics_elapsed_time[2] < t2.metrics_elapsed_time[1] AND
+ t2.metrics_elapsed_time[2] < t3.metrics_elapsed_time[1] ,
+ 'metrics_elapsed_time should be cumulative for each trial.'
+) FROM (SELECT * FROM automl_output_info WHERE mst_key=1) t1,
+ (SELECT * FROM automl_output_info WHERE mst_key=3) t2,
+ (SELECT * FROM automl_output_info WHERE mst_key=5) t3;
+
+--------------------------- HYPERBAND TEST CASES ---------------------------
+
+-- test table dimensions / happy path with default automl_params
+DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyp', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$);
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=15, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=15, 'The length of table does not match with the inputs') FROM automl_output_info;
+SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output;
+SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output_info;
--- test automl_params vals {R, eta, skip_last}
+-- Validate model output summary table
+SELECT assert(
+ source_table = 'iris_data_packed' AND
+ validation_table IS NULL AND
+ model = 'automl_output' AND
+ model_info = 'automl_output_info' AND
+ dependent_varname = 'class_text' AND
+ independent_varname = 'attributes' AND
+ model_arch_table = 'iris_model_arch' AND
+ model_selection_table = 'automl_mst_table' AND
+ automl_method = 'hyperband' AND
+ automl_params = 'R=6, eta=3, skip_last=0' AND
+ random_state IS NULL AND
+ object_table IS NULL AND
+ use_gpus = FALSE AND
+ metrics_compute_frequency = 6 AND
+ name IS NULL AND
+ description IS NULL AND
+ start_training_time < now() AND
+ end_training_time < now() AND
+ madlib_version IS NOT NULL AND
+ num_classes = 3 AND
+ class_values = '{Iris-setosa,Iris-versicolor,Iris-virginica}' AND
+ dependent_vartype = 'character varying' AND
+ normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
+) FROM (SELECT * FROM automl_output_summary) summary;
+
+-- Validate output info table for s and i NOT NULL
+SELECT assert(
+ metrics_iters IS NOT NULL AND
+ s = ANY(ARRAY[0,1]) AND
+ i = ANY(ARRAY[0,1]) , 'Invalid metrics_iters, s and i value in output info table. Actual:' || __to_char(info)
+) FROM (SELECT * FROM automl_output_info) info;
+
+-- Validate mst summary table
+SELECT assert(
+ model_arch_table = 'iris_model_arch' AND
+ object_table IS NULL , 'mst summary table validation failed. Actual:' || __to_char(summary)
+) FROM (SELECT * FROM automl_mst_table_summary) summary;
+
+-- test invalid source table
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=10, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('invalid_source_table', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid source table');
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=15, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=15, 'The length of table does not match with the inputs') FROM automl_output_info;
+-- test preexisting output table
+CREATE TABLE automl_output(a int);
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for preexisting output table');
+-- test preexisting selection table
+CREATE TABLE automl_mst_table(a int);
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for preexisting selection table');
+
+-- test test invalid model id
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=5, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[2,-1], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid model id');
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output_info;
+-- test invalid automl params {R, eta, skip_last}
+DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
+ automl_mst_table_summary;
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=2, eta=3, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid automl params: R');
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=10, eta=4, skip_last=1', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
-
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=4, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=4, 'The length of table does not match with the inputs') FROM automl_output_info;
+ 'hyperband', 'R=9, eta=1, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid automl params: eta');
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=5, eta=5, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperband', 'R=9, eta=3, skip_last=3', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid automl params: skip_last');
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
-SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=7, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=7, 'The length of table does not match with the inputs') FROM automl_output_info;
+-- test invalid object table
+DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
+ automl_mst_table_summary;
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=9, eta=3, skip_last=0', NULL, 'invalid_object_table', FALSE, NULL, NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid object table');
+-- test invalid validation table
+DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
+ automl_mst_table_summary;
+SELECT assert(trap_error($TRAP$
+ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
+ ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
+ 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
+ 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
+ 'hyperband', 'R=9, eta=3, skip_last=0', NULL, NULL, FALSE, 'invalid_validation_table', NULL, NULL, NULL);
+$TRAP$)=1, 'Should error out for invalid validation table');
+
+-- test automl_params vals {R, eta, skip_last}
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=2', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperband', 'R=5, eta=5, skip_last=1', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=9, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=9, 'The length of table does not match with the inputs') FROM automl_output_info;
+SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output;
+SELECT assert(COUNT(*)=5, 'The length of table does not match with the inputs') FROM automl_output_info;
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
automl_mst_table_summary;
@@ -293,24 +458,13 @@ SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=11, eta=2, skip_last=3', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperband', 'R=5, eta=5, skip_last=0', NULL, NULL, FALSE, NULL, NULL, NULL, NULL);
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_mst_table_summary;
SELECT assert(COUNT(*)=1, 'The length of table does not match with the inputs') FROM automl_output_summary;
-SELECT assert(COUNT(*)=8, 'The length of table does not match with the inputs') FROM automl_output;
-SELECT assert(COUNT(*)=8, 'The length of table does not match with the inputs') FROM automl_output_info;
-
--- test name and description
-DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table,
- automl_mst_table_summary;
-SELECT madlib_keras_automl('iris_data_packed', 'automl_output', 'iris_model_arch', 'automl_mst_table',
- ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
- 'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
- 'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=11, eta=2, skip_last=3', NULL, NULL, FALSE, NULL, NULL, 'test1', 'test1 descr');
-SELECT assert(name='test1' AND description='test1 descr',
- 'invalid name/description') FROM (SELECT * FROM automl_output_summary) summary;
+SELECT assert(COUNT(*)=7, 'The length of table does not match with the inputs') FROM automl_output;
+SELECT assert(COUNT(*)=7, 'The length of table does not match with the inputs') FROM automl_output_info;
-- test config reproducibility
DROP TABLE IF EXISTS automl_output1, automl_output1_info, automl_output1_summary, automl_mst_table1,
@@ -319,7 +473,7 @@ SELECT madlib_keras_automl('iris_data_packed', 'automl_output1', 'iris_model_arc
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=1', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperband', 'R=3, eta=2, skip_last=1', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
DROP TABLE IF EXISTS automl_output2, automl_output2_info, automl_output2_summary, automl_mst_table2,
automl_mst_table2_summary;
@@ -327,7 +481,7 @@ SELECT madlib_keras_automl('iris_data_packed', 'automl_output2', 'iris_model_arc
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=1', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
+ 'hyperband', 'R=3, eta=2, skip_last=1', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
DROP TABLE IF EXISTS automl_output3, automl_output3_info, automl_output3_summary, automl_mst_table3,
automl_mst_table3_summary;
@@ -335,16 +489,21 @@ SELECT madlib_keras_automl('iris_data_packed', 'automl_output3', 'iris_model_arc
ARRAY[1,2], $${'loss': ['categorical_crossentropy'], 'optimizer_params_list': [ {'optimizer': ['Adagrad', 'Adam'],
'lr': [0.9, 0.95, 'log'], 'epsilon': [0.3, 0.5, 'log_near_one']}, {'optimizer': ['Adam', 'SGD'],
'lr': [0.6, 0.65, 'log']} ], 'metrics':['accuracy'] }$$, $${'batch_size': [2, 4], 'epochs': [3]}$$,
- 'hyperband', 'R=9, eta=3, skip_last=1', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
-
-SELECT assert(model_id=(SELECT model_id FROM automl_output2_info WHERE mst_key=7) AND
- compile_params=(SELECT compile_params FROM automl_output2_info WHERE mst_key=7) AND
- fit_params=(SELECT fit_params FROM automl_output2_info WHERE mst_key=7), 'invalid config uniformity')
-FROM (SELECT model_id, compile_params, fit_params FROM automl_output1_info WHERE mst_key=7) output1;
-SELECT assert(model_id=(SELECT model_id FROM automl_output2_info WHERE mst_key=7) AND
- compile_params=(SELECT compile_params FROM automl_output2_info WHERE mst_key=7) AND
- fit_params=(SELECT fit_params FROM automl_output2_info WHERE mst_key=7), 'invalid config uniformity')
-FROM (SELECT model_id, compile_params, fit_params FROM automl_output3_info WHERE mst_key=7) output3;
+ 'hyperband', 'R=3, eta=2, skip_last=1', 42, NULL, FALSE, NULL, NULL, NULL, NULL);
+
+SELECT assert(model_id=(SELECT model_id FROM automl_output2_info WHERE mst_key=2) AND
+ compile_params=(SELECT compile_params FROM automl_output2_info WHERE mst_key=2) AND
+ fit_params=(SELECT fit_params FROM automl_output2_info WHERE mst_key=2), 'invalid config uniformity')
+FROM (SELECT model_id, compile_params, fit_params FROM automl_output1_info WHERE mst_key=2) output1;
+SELECT assert(model_id=(SELECT model_id FROM automl_output2_info WHERE mst_key=2) AND
+ compile_params=(SELECT compile_params FROM automl_output2_info WHERE mst_key=2) AND
+ fit_params=(SELECT fit_params FROM automl_output2_info WHERE mst_key=2), 'invalid config uniformity')
+FROM (SELECT model_id, compile_params, fit_params FROM automl_output3_info WHERE mst_key=2) output3;
+
+DROP TABLE IF EXISTS automl_output1, automl_output1_info, automl_output1_summary, automl_mst_table1,
+ automl_mst_table1_summary, automl_output2, automl_output2_info, automl_output2_summary, automl_mst_table2,
+ automl_mst_table2_summary, automl_output3, automl_output3_info, automl_output3_summary, automl_mst_table3,
+ automl_mst_table3_summary;
--------------------------- HYPERBAND SCHEDULE TEST CASES ---------------------------
-- Testing happy path with default values
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_automl.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_automl.py_in
index 15b3851..edb12c4 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_automl.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_automl.py_in
@@ -206,5 +206,79 @@ class HyperbandScheduleTestCase(unittest.TestCase):
def tearDown(self):
self.module_patcher.stop()
+
+
+class AutoMLHyperoptTestCase(unittest.TestCase):
+ def setUp(self):
+ # The side effects of this class(writing to the output table) are not
+ # tested here. They are tested in dev-check.
+ self.plpy_mock = Mock(spec='error')
+ patches = {
+ 'plpy': plpy
+ }
+
+ self.plpy_mock_execute = MagicMock()
+ plpy.execute = self.plpy_mock_execute
+
+ self.module_patcher = patch.dict('sys.modules', patches)
+ self.module_patcher.start()
+ import deep_learning.madlib_keras_automl
+ self.module = deep_learning.madlib_keras_automl
+
+ from deep_learning.madlib_keras_automl import AutoMLHyperopt
+ self.seg_num_mock = Mock()
+
+ class FakeAutoMLHyperopt(AutoMLHyperopt):
+ def __init__(self, *args):
+ pass
+ self.module.get_seg_number = self.seg_num_mock
+
+ self.subject = FakeAutoMLHyperopt
+
+ def test_get_configs_list_models_less_than_segments(self):
+ automl_hyperopt = self.subject()
+ configs = automl_hyperopt.get_configs_list(1,3)
+ self.assertEquals([(1,1)], configs)
+
+ def test_get_configs_list_models_equal_segments(self):
+ automl_hyperopt = self.subject()
+ configs = automl_hyperopt.get_configs_list(3,3)
+ self.assertEquals([(1,3)], configs)
+
+ def test_get_configs_list_last_bucket_models_less_than_half_segments(self):
+ automl_hyperopt = self.subject()
+ # Last bucket num models < 1/2 num workers
+ configs = automl_hyperopt.get_configs_list(81,20)
+ self.assertEquals([(1, 20), (21, 40), (41, 60), (61, 81)], configs)
+
+ def test_get_configs_list_last_bucket_models_greater_than_half_segments(self):
+ automl_hyperopt = self.subject()
+ # Last bucket num models > 1/2 num workers
+ configs = automl_hyperopt.get_configs_list(20,3)
+ self.assertEquals([(1, 3), (4, 6), (7, 9), (10, 12), (13, 15), (16, 18),(19, 20)], configs)
+
+ def test_get_configs_list_last_bucket_models_equal_half_segments(self):
+ automl_hyperopt = self.subject()
+ # Last bucket num models = 1/2 num workers
+ configs = automl_hyperopt.get_configs_list(90,20)
+ self.assertEquals([(1, 20), (21, 40), (41, 60), (61, 80),(81,90)], configs)
+
+ def test_get_num_segments_all_segments(self):
+ automl_hyperopt = self.subject()
+ automl_hyperopt.source_table = 'dummy_table'
+ self.plpy_mock_execute.return_value = [{'distribution_rules': 'all_segments'}]
+ self.seg_num_mock.return_value = 3
+ self.assertEquals(3, automl_hyperopt.get_num_segments())
+
+ def test_get_num_segments_array_value(self):
+ automl_hyperopt = self.subject()
+ automl_hyperopt.source_table = 'dummy_table'
+ # return list of segment ids as distribution_rules
+ self.plpy_mock_execute.return_value = [{'distribution_rules': [3,1]}]
+ self.assertEquals(2, automl_hyperopt.get_num_segments())
+
+ def tearDown(self):
+ self.module_patcher.stop()
+
if __name__ == '__main__':
unittest.main()
diff --git a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
index 1a2f61f..7de9868 100644
--- a/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
+++ b/src/ports/postgres/modules/deep_learning/test/unit_tests/test_madlib_keras_model_selection_table.py_in
@@ -38,7 +38,8 @@ class GenerateModelSelectionConfigsTestCase(unittest.TestCase):
# tested here. They are tested in dev-check.
self.plpy_mock = Mock(spec='error')
patches = {
- 'plpy': plpy
+ 'plpy': plpy,
+ 'utilities.mean_std_dev_calculator': Mock()
}
self.plpy_mock_execute = MagicMock()
@@ -354,7 +355,8 @@ class LoadModelSelectionTableTestCase(unittest.TestCase):
# tested here. They are tested in dev-check.
self.plpy_mock = Mock(spec='error')
patches = {
- 'plpy': plpy
+ 'plpy': plpy,
+ 'utilities.mean_std_dev_calculator': Mock()
}
self.plpy_mock_execute = MagicMock()
@@ -478,7 +480,8 @@ class MstLoaderInputValidatorTestCase(unittest.TestCase):
# tested here. They are tested in dev-check.
self.plpy_mock = Mock(spec='error')
patches = {
- 'plpy': plpy
+ 'plpy': plpy,
+ 'utilities.mean_std_dev_calculator': Mock()
}
self.plpy_mock_execute = MagicMock()
diff --git a/src/ports/postgres/modules/utilities/utilities.py_in b/src/ports/postgres/modules/utilities/utilities.py_in
index b228d68..eb507d9 100644
--- a/src/ports/postgres/modules/utilities/utilities.py_in
+++ b/src/ports/postgres/modules/utilities/utilities.py_in
@@ -1,9 +1,11 @@
import collections
+from datetime import datetime
import re
import time
import random
from distutils.util import strtobool
+# import numpy as np
from validate_args import _get_table_schema_names
from validate_args import cols_in_tbl_valid
@@ -1317,3 +1319,8 @@ def get_schema(tbl_str):
else:
return None
+# -------------------------------------------------------------------------------
+
+def get_current_timestamp(format):
+ """Gets current time stamp in the specified format string"""
+ return datetime.fromtimestamp(time.time()).strftime(format)