You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@madlib.apache.org by nk...@apache.org on 2021/01/20 03:08:05 UTC
[madlib] branch master updated: DL: Fix hyperband
metrics_elapsed_time calculation (#528)
This is an automated email from the ASF dual-hosted git repository.
nkak pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/madlib.git
The following commit(s) were added to refs/heads/master by this push:
new 841f54d DL: Fix hyperband metrics_elapsed_time calculation (#528)
841f54d is described below
commit 841f54d459b079d46f215647ea14b1e88588277e
Author: kaknikhil <nk...@vmware.com>
AuthorDate: Tue Jan 19 19:07:57 2021 -0800
DL: Fix hyperband metrics_elapsed_time calculation (#528)
* DL: Fix hyperband metrics_elapsed_time calculation
JIRA: MADLIB-1456
For hyperband, the metrics_elapsed_time column was not cumulative. This commit
adds code to make it cumulative
Also fixed example in user docs
Co-authored-by: Ekta Khanna <ek...@vmware.com>
Co-authored-by: Frank McQuillan <fm...@pivotal.io>
---
.../deep_learning/madlib_keras_automl.sql_in | 103 ++++++++++-----------
.../madlib_keras_automl_hyperband.py_in | 8 +-
.../deep_learning/test/madlib_keras_automl.sql_in | 23 +++++
3 files changed, 76 insertions(+), 58 deletions(-)
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in
index f692f13..66b1a91 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl.sql_in
@@ -977,7 +977,7 @@ $$
<h4>Hyperband</h4>
--# Print Hyperband schedule for example input parameters:
+-# Print Hyperband schedule for example input parameters 'R=9' and 'eta=3':
<pre class="example">
DROP TABLE IF EXISTS hb_schedule;
SELECT madlib.hyperband_schedule ('hb_schedule',
@@ -1006,7 +1006,7 @@ SELECT * FROM hb_schedule ORDER BY s DESC, i;
0 | 0 | 5 | 81
(15 rows)
</pre>
--# Run Hyperband method:
+-# Run Hyperband method with 'R=9' and 'eta=3':
<pre class="example">
DROP TABLE IF EXISTS automl_output, automl_output_info, automl_output_summary, automl_mst_table, automl_mst_table_summary;
SELECT madlib.madlib_keras_automl('iris_train_packed', -- source table
@@ -1055,8 +1055,8 @@ use_gpus | f
metrics_compute_frequency | 1
name |
description |
-start_training_time | 2020-10-23 00:20:52
-end_training_time | 2020-10-23 00:22:19
+start_training_time | 2021-01-16 01:20:17
+end_training_time | 2021-01-16 01:21:47
madlib_version | 1.18.0-dev
num_classes | 3
class_values | {Iris-setosa,Iris-versicolor,Iris-virginica}
@@ -1068,75 +1068,66 @@ normalizing_const | 1
SELECT * FROM automl_output_info ORDER BY validation_metrics_final DESC, validation_loss_final LIMIT 3;
</pre>
<pre class="result">
--[ RECORD 1]----------------------------------------------------------------------------------------------------------
-mst_key | 13
+-[ RECORD 1 ]------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------
+mst_key | 15
model_id | 1
-compile_params | optimizer='Adam(lr=0.09102824394462919)',metrics=['accuracy'],loss='categorical_crossentropy'
-fit_params | epochs=1,batch_size=4
+compile_params | optimizer='Adam(lr=0.005948073640447284)',metrics=['accuracy'],loss='categorical_crossentropy'
+fit_params | epochs=1,batch_size=8
model_type | madlib_keras
model_size | 0.7900390625
-metrics_elapsed_time | {5.37390279769897,11.0799419879913,16.6234488487244,22.406044960022,28.1228229999542,33.9054269790649,39.4304218292236,45.18506193161
-01,50.8772490024567}
+metrics_elapsed_time | {41.9598820209503,47.7600600719452,53.5559930801392,59.2904281616211,65.0303740501404,70.910637140274,76.6586999893188,82.3321261405945,88.0252130031586}
metrics_type | {accuracy}
loss_type | categorical_crossentropy
-training_metrics_final | 0.966666638851166
-training_loss_final | 0.117527179419994
-training_metrics | {0.658333361148834,0.975000023841858,0.616666674613953,0.800000011920929,0.975000023841858,0.633333325386047,0.783333361148834,0.9666
-66638851166,0.966666638851166}
-training_loss | {0.495927810668945,0.18766151368618,0.515772044658661,0.352419972419739,0.0683904364705086,0.749827742576599,0.778484106063843,0.1788
-07646036148,0.117527179419994}
-validation_metrics_final | 1
-validation_loss_final | 0.0773465484380722
-validation_metrics | {0.800000011920929,0.933333337306976,0.800000011920929,0.800000011920929,0.966666638851166,0.800000011920929,0.833333313465118,1,1}
-validation_loss | {0.353509128093719,0.181772708892822,0.291709125041962,0.319768697023392,0.0782377645373344,0.42935499548912,0.600821077823639,0.1398
-00950884819,0.0773465484380722}
+training_metrics_final | 0.975000023841858
+training_loss_final | 0.174209594726562
+training_metrics | {0.683333337306976,0.683333337306976,0.816666662693024,0.791666686534882,0.966666638851166,0.850000023841858,0.966666638851166,0.966666638851166,0.975000023841858}
+training_loss | {0.658287584781647,0.56329345703125,0.489711940288544,0.417204052209854,0.333063006401062,0.325938105583191,0.237209364771843,0.216858893632889,0.174209594726562}
+validation_metrics_final | 0.933333337306976
+validation_loss_final | 0.282542854547501
+validation_metrics | {0.600000023841858,0.600000023841858,0.733333349227905,0.733333349227905,0.899999976158142,0.800000011920929,0.933333337306976,0.899999976158142,0.933333337306976}
+validation_loss | {0.844917356967926,0.739157736301422,0.651688754558563,0.567608654499054,0.458681106567383,0.461867392063141,0.344642341136932,0.335768848657608,0.282542854547501}
metrics_iters | {5,6,7,8,9,10,11,12,13}
s | 0
i | 0
--[ RECORD 2]----------------------------------------------------------------------------------------------------------
-mst_key | 12
-model_id | 2
-compile_params | optimizer='RMSprop(lr=0.0038913960350146193)',metrics=['accuracy'],loss='categorical_crossentropy'
-fit_params | epochs=1,batch_size=4
+-[ RECORD 2 ]------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------
+mst_key | 10
+model_id | 1
+compile_params | optimizer='RMSprop(lr=0.01152123686692268)',metrics=['accuracy'],loss='categorical_crossentropy'
+fit_params | epochs=1,batch_size=8
model_type | madlib_keras
-model_size | 1.2197265625
-metrics_elapsed_time | {6.42797207832336,13.1542639732361,19.5103330612183,5.07128381729126,10.5596950054169,16.2247838973999,22.1064488887787,27.7473468780
-518,33.3738968372345,39.1290938854218,44.8826239109039,50.5644388198853}
+model_size | 0.7900390625
+metrics_elapsed_time | {21.1628739833832,27.9904689788818,34.9025909900665}
metrics_type | {accuracy}
loss_type | categorical_crossentropy
-training_metrics_final | 0.949999988079071
-training_loss_final | 0.107505217194557
-training_metrics | {0.666666686534882,0.949999988079071,0.958333313465118,0.949999988079071,0.774999976158142,0.941666662693024,0.941666662693024,0.9583
-33313465118,0.858333349227905,0.941666662693024,0.958333313465118,0.949999988079071}
-training_loss | {0.807353079319,0.531827747821808,0.375816851854324,0.29875060915947,0.358659148216248,0.192024797201157,0.200978621840477,0.15286296
-6060638,0.272547781467438,0.125148341059685,0.11623315513134,0.107505217194557}
-validation_metrics_final | 0.966666638851166
-validation_loss_final | 0.0540979467332363
-validation_metrics | {0.699999988079071,0.899999976158142,0.966666638851166,0.966666638851166,0.899999976158142,1,0.899999976158142,0.966666638851166,0.89
-9999976158142,0.966666638851166,1,0.966666638851166}
-validation_loss | {0.819778800010681,0.472518295049667,0.300146490335464,0.222854107618332,0.225204601883888,0.147142887115479,0.192571476101875,0.1301
-05406045914,0.178782090544701,0.0740523263812065,0.0714136436581612,0.0540979467332363}
-metrics_iters | {2,3,4,5,6,7,8,9,10,11,12,13}
+training_metrics_final | 0.933333337306976
+training_loss_final | 0.239687830209732
+training_metrics | {0.699999988079071,0.699999988079071,0.933333337306976}
+training_loss | {0.600760638713837,0.386314034461975,0.239687830209732}
+validation_metrics_final | 0.899999976158142
+validation_loss_final | 0.369663149118423
+validation_metrics | {0.533333361148834,0.600000023841858,0.899999976158142}
+validation_loss | {0.723896682262421,0.539595663547516,0.369663149118423}
+metrics_iters | {2,3,4}
s | 1
-i | 1
--[ RECORD 3]----------------------------------------------------------------------------------------------------------
-mst_key | 6
+i | 0
+-[ RECORD 3 ]------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------
+mst_key | 2
model_id | 1
-compile_params | optimizer='Adam(lr=0.02358545238214664)',metrics=['accuracy'],loss='categorical_crossentropy'
-fit_params | epochs=1,batch_size=8
+compile_params | optimizer='RMSprop(lr=0.005464438486993435)',metrics=['accuracy'],loss='categorical_crossentropy'
+fit_params | epochs=1,batch_size=4
model_type | madlib_keras
model_size | 0.7900390625
-metrics_elapsed_time | {10.6679489612579,6.12874889373779,12.8602039813995,19.2172629833221}
+metrics_elapsed_time | {11.6164019107819,20.9570059776306,27.7901480197906,34.7061359882355}
metrics_type | {accuracy}
loss_type | categorical_crossentropy
-training_metrics_final | 0.966666638851166
-training_loss_final | 0.367920279502869
-training_metrics | {0.658333361148834,0.658333361148834,0.891666650772095,0.966666638851166}
-training_loss | {0.744447708129883,0.627880990505219,0.487682670354843,0.367920279502869}
-validation_metrics_final | 0.933333337306976
-validation_loss_final | 0.280433148145676
-validation_metrics | {0.699999988079071,0.699999988079071,0.933333337306976,0.933333337306976}
-validation_loss | {0.5818150639534,0.465440601110458,0.350821226835251,0.280433148145676}
+training_metrics_final | 0.925000011920929
+training_loss_final | 0.17901936173439
+training_metrics | {0.949999988079071,0.883333325386047,0.958333313465118,0.925000011920929}
+training_loss | {0.547602951526642,0.321837723255157,0.197886273264885,0.17901936173439}
+validation_metrics_final | 0.866666674613953
+validation_loss_final | 0.325421392917633
+validation_metrics | {0.866666674613953,0.800000011920929,0.899999976158142,0.866666674613953}
+validation_loss | {0.723824441432953,0.462396681308746,0.326263695955276,0.325421392917633}
metrics_iters | {1,2,3,4}
s | 2
i | 1
diff --git a/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperband.py_in b/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperband.py_in
index d44c3ea..d4f5211 100644
--- a/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperband.py_in
+++ b/src/ports/postgres/modules/deep_learning/madlib_keras_automl_hyperband.py_in
@@ -19,7 +19,7 @@
import math
import plpy
-
+import time
from madlib_keras_automl import KerasAutoML, AutoMLConstants
from utilities.utilities import get_current_timestamp, get_seg_number, get_segments_per_host, \
unique_string, add_postfix, extract_keyvalue_params, _assert, _assert_equal, rename_table, \
@@ -228,6 +228,7 @@ class AutoMLHyperband(KerasAutoML):
i_dict[mst_key] = -1
# outer loop on diagonal
+ metrics_elapsed_time_offset = 0
for i in range((self.s_max+1) - int(self.skip_last)):
# inner loop on s desc
temp_lst = []
@@ -249,11 +250,14 @@ class AutoMLHyperband(KerasAutoML):
i_dict[k[ModelSelectionSchema.MST_KEY]] += 1
self.warm_start = int(i != 0)
mcf = self.metrics_compute_frequency if self._is_valid_metrics_compute_frequency(num_iterations) else None
+ start_time = time.time()
with SetGUC("plan_cache_mode", "force_generic_plan"):
model_training = FitMultipleModel(self.schema_madlib, self.source_table, AutoMLConstants.MODEL_OUTPUT_TABLE,
AutoMLConstants.MST_TABLE, num_iterations, self.use_gpus,
- self.validation_table, mcf, self.warm_start, self.name, self.description)
+ self.validation_table, mcf, self.warm_start, self.name, self.description,
+ metrics_elapsed_time_offset=metrics_elapsed_time_offset)
model_training.fit_multiple_model()
+ metrics_elapsed_time_offset += time.time() - start_time
self.update_model_output_table()
self.update_model_output_info_table(i, initial_vals)
diff --git a/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in b/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
index da9fb8a..cbb6dd0 100644
--- a/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
+++ b/src/ports/postgres/modules/deep_learning/test/madlib_keras_automl.sql_in
@@ -331,6 +331,29 @@ SELECT assert(
normalizing_const = 1, 'Output summary table validation failed. Actual:' || __to_char(summary)
) FROM (SELECT * FROM automl_output_summary) summary;
+-- Validate metrics_elapsed_time
+-- We know that the schedule for the above automl query looks like
+-- s | i | n_i | r_i
+-- ---+---+-----+-----
+-- 1 | 0 | 3 | 2
+-- 1 | 1 | 1 | 6
+-- 0 | 0 | 2 | 6
+-- So we can make 3 assertions:
+-- 1. The metrics_elapsed_time array should have cumulative time for each mst
+-- 2. Within the same bracket, we know that one mst is run for 8 iterations and
+-- the other two are run for 2 iterations. So we can assert that the 3rd iteration
+-- time for (s=1,i=1) > the 2nd iteration time of any of the other two (s=1,i=0)
+-- 3. Within different brackets (s=1,i=0) and (s=0,i=0), we know that the 1st iteration of
+-- the former > last/2nd iteration time of the latter
+
+SELECT assert(metrics_elapsed_time = ARRAY(SELECT unnest(metrics_elapsed_time) ORDER BY 1),
+ 'metrics_elapsed_time is not cumulative over iterations for one or more msts') FROM automl_output_info metrics_iters;
+SELECT assert(t1.metrics_elapsed_time[2] < t2.metrics_elapsed_time[3] , 'metrics_elapsed_time is not
+ cumulative within the same bracket.') FROM (SELECT * FROM automl_output_info WHERE s=1 and i = 0) t1, (SELECT * FROM automl_output_info WHERE s=1 and i = 1) t2;
+SELECT assert(t1.metrics_elapsed_time[2] < t2.metrics_elapsed_time[1] , 'metrics_elapsed_time is not
+ cumulative between two brackets.') FROM (SELECT * FROM automl_output_info WHERE s=1 and i = 0) t1,
+ (SELECT * FROM automl_output_info WHERE s=0 and i =0) t2;
+
-- Validate output info table for s and i NOT NULL
SELECT assert(
metrics_iters IS NOT NULL AND