You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@madlib.apache.org by "Frank McQuillan (Jira)" <ji...@apache.org> on 2020/10/12 23:07:00 UTC
[jira] [Updated] (MADLIB-1456) Hyperband cumulative output time is
not correct
[ https://issues.apache.org/jira/browse/MADLIB-1456?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Frank McQuillan updated MADLIB-1456:
------------------------------------
Description:
{code}
SELECT madlib.madlib_keras_automl('cifar10_train_packed',
'automl_cifar10_output',
'model_arch_library',
'automl_cifar10_mst_table',
ARRAY[1,2,3],
$${'loss': ['categorical_crossentropy'],
'optimizer_params_list': [
{'optimizer': ['Adam'],'lr': [0.0001, 0.01, 'log']},
{'optimizer': ['RMSprop'],'lr': [0.0001, 0.01, 'log'], 'decay': [1.0e-7, 1.0e-5, 'log']},
{'optimizer': ['SGD'],'lr': [0.0001, 0.01, 'log'], 'momentum': [0.9, 0.99,'log_near_one']}],
'metrics':['accuracy']}$$,
$${'batch_size': [64,128], 'epochs': [1]}$$,
'hyperband',
'R=27, eta=3, skip_last=1',
NULL, -- random state
NULL, -- object table
FALSE, -- use GPUs
'cifar10_val_packed', -- validation table
3, -- metrics compute freq
NULL, -- name
NULL); -- descr
{code}
produces
{code}
select mst_key, metrics_iters, metrics_elapsed_time from hb_cifar10_output_info_r27e3skip1run1 order by mst_key, metrics_iters;
mst_key | metrics_iters | metrics_elapsed_time
---------+---------------+----------------------------------------------------------------------------------------
1 | {1} | {5639.45646381378}
2 | {1,4} | {5499.1161878109,10417.6853170395}
3 | {1} | {5647.45680689812}
4 | {1,4} | {5581.32227087021,10422.9603590965}
5 | {1} | {5473.09724283218}
6 | {1} | {5537.51781392097}
7 | {1} | {5468.15699982643}
8 | {1} | {5597.43572378159}
9 | {1,4} | {5618.64315199852,10291.4588999748}
10 | {1} | {5450.7096657753}
11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
12 | {1} | {5674.29211378098}
13 | {1} | {5542.49673891068}
14 | {1} | {5682.2088098526}
15 | {1,4,7,10,13} | {5610.54115080833,10365.9494991302,7378.03450798988,14876.2571978569,22414.0461809635}
16 | {1} | {5563.48544478416}
17 | {1,4,7,10,13} | {5661.10293698311,10398.6006779671,7373.00771188736,14871.3066940308,22409.0431649685}
18 | {1} | {5626.78051185608}
19 | {1} | {5656.17028284073}
20 | {1} | {5524.58638978004}
21 | {1} | {5555.59004878998}
22 | {1} | {5605.62219786644}
23 | {1} | {5494.16033291817}
24 | {1,4} | {5512.02432179451,10360.8255741596}
25 | {1,4} | {5455.62597084045,10296.5835280418}
26 | {1,4} | {5589.49808478355,10337.8003959656}
27 | {1} | {5576.40678977966}
28 | {4,7,10,13} | {10393.5381979942,7413.2452878952,14912.529984951,22449.6516678333}
29 | {4} | {10319.5200190544}
30 | {4} | {10451.2104020119}
31 | {4,7,10,13} | {10328.5663330555,7399.8391699791,14899.122522831,22436.2282240391}
32 | {4} | {10437.2220571041}
33 | {4} | {10379.5259420872}
34 | {4} | {10346.9251220226}
35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
36 | {4} | {10412.5241000652}
37 | {7,10,13} | {7421.8063390255,14921.1340680122,22459.3360388279}
38 | {7,10,13} | {7469.96136283875,14970.6747159958,22509.476047039}
39 | {7,10,13} | {7461.39016890526,14962.0603349209,22500.7105178833}
40 | {7,10,13} | {7435.0908639431,14935.0127699375,22472.9902210236}
41 | {7,10,13} | {7391.56045103073,14890.2629489899,22427.4269089699}
42 | {7,10,13} | {7354.62691497803,14852.9146559238,22389.92344594}
(42 rows)
{code}
Note that for times with 7 cumulative time is wrong, here are 2 exs:
{code}
11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
and
35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
{code}
The expected schedule is:
{code}
DROP TABLE IF EXISTS hb_schedule;
SELECT madlib.hyperband_schedule ('hb_schedule',
27,
3,
1);
SELECT * FROM hb_schedule ORDER BY s DESC, i;
s | i | n_i | r_i
---+---+-----+-----
3 | 0 | 27 | 1
3 | 1 | 9 | 3
3 | 2 | 3 | 9
2 | 0 | 9 | 3
2 | 1 | 3 | 9
1 | 0 | 6 | 9
(6 rows)
{code}
was:
{code}
SELECT madlib.madlib_keras_automl('cifar10_train_packed',
'automl_cifar10_output',
'model_arch_library',
'automl_cifar10_mst_table',
ARRAY[1,2,3],
$${'loss': ['categorical_crossentropy'],
'optimizer_params_list': [
{'optimizer': ['Adam'],'lr': [0.0001, 0.01, 'log']},
{'optimizer': ['RMSprop'],'lr': [0.0001, 0.01, 'log'], 'decay': [1.0e-7, 1.0e-5, 'log']},
{'optimizer': ['SGD'],'lr': [0.0001, 0.01, 'log'], 'momentum': [0.9, 0.99,'log_near_one']}],
'metrics':['accuracy']}$$,
$${'batch_size': [64,128], 'epochs': [1]}$$,
'hyperband',
'R=27, eta=3, skip_last=1',
NULL, -- random state
NULL, -- object table
FALSE, -- use GPUs
'cifar10_val_packed', -- validation table
3, -- metrics compute freq
NULL, -- name
NULL); -- descr
{code}
produces
{code}
select mst_key, metrics_iters, metrics_elapsed_time from hb_cifar10_output_info_r27e3skip1run1 order by mst_key, metrics_iters;
mst_key | metrics_iters | metrics_elapsed_time
---------+---------------+----------------------------------------------------------------------------------------
1 | {1} | {5639.45646381378}
2 | {1,4} | {5499.1161878109,10417.6853170395}
3 | {1} | {5647.45680689812}
4 | {1,4} | {5581.32227087021,10422.9603590965}
5 | {1} | {5473.09724283218}
6 | {1} | {5537.51781392097}
7 | {1} | {5468.15699982643}
8 | {1} | {5597.43572378159}
9 | {1,4} | {5618.64315199852,10291.4588999748}
10 | {1} | {5450.7096657753}
11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
12 | {1} | {5674.29211378098}
13 | {1} | {5542.49673891068}
14 | {1} | {5682.2088098526}
15 | {1,4,7,10,13} | {5610.54115080833,10365.9494991302,7378.03450798988,14876.2571978569,22414.0461809635}
16 | {1} | {5563.48544478416}
17 | {1,4,7,10,13} | {5661.10293698311,10398.6006779671,7373.00771188736,14871.3066940308,22409.0431649685}
18 | {1} | {5626.78051185608}
19 | {1} | {5656.17028284073}
20 | {1} | {5524.58638978004}
21 | {1} | {5555.59004878998}
22 | {1} | {5605.62219786644}
23 | {1} | {5494.16033291817}
24 | {1,4} | {5512.02432179451,10360.8255741596}
25 | {1,4} | {5455.62597084045,10296.5835280418}
26 | {1,4} | {5589.49808478355,10337.8003959656}
27 | {1} | {5576.40678977966}
28 | {4,7,10,13} | {10393.5381979942,7413.2452878952,14912.529984951,22449.6516678333}
29 | {4} | {10319.5200190544}
30 | {4} | {10451.2104020119}
31 | {4,7,10,13} | {10328.5663330555,7399.8391699791,14899.122522831,22436.2282240391}
32 | {4} | {10437.2220571041}
33 | {4} | {10379.5259420872}
34 | {4} | {10346.9251220226}
35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
36 | {4} | {10412.5241000652}
37 | {7,10,13} | {7421.8063390255,14921.1340680122,22459.3360388279}
38 | {7,10,13} | {7469.96136283875,14970.6747159958,22509.476047039}
39 | {7,10,13} | {7461.39016890526,14962.0603349209,22500.7105178833}
40 | {7,10,13} | {7435.0908639431,14935.0127699375,22472.9902210236}
41 | {7,10,13} | {7391.56045103073,14890.2629489899,22427.4269089699}
42 | {7,10,13} | {7354.62691497803,14852.9146559238,22389.92344594}
(42 rows)
{code}
Note that for times with 7 cumulative time is wrong
{code}
11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
and
35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
{code}
The expected schedule is:
{code}
DROP TABLE IF EXISTS hb_schedule;
SELECT madlib.hyperband_schedule ('hb_schedule',
27,
3,
1);
SELECT * FROM hb_schedule ORDER BY s DESC, i;
s | i | n_i | r_i
---+---+-----+-----
3 | 0 | 27 | 1
3 | 1 | 9 | 3
3 | 2 | 3 | 9
2 | 0 | 9 | 3
2 | 1 | 3 | 9
1 | 0 | 6 | 9
(6 rows)
{code}
> Hyperband cumulative output time is not correct
> -----------------------------------------------
>
> Key: MADLIB-1456
> URL: https://issues.apache.org/jira/browse/MADLIB-1456
> Project: Apache MADlib
> Issue Type: Bug
> Components: Deep Learning
> Reporter: Frank McQuillan
> Priority: Major
> Fix For: v1.18.0
>
>
> {code}
> SELECT madlib.madlib_keras_automl('cifar10_train_packed',
> 'automl_cifar10_output',
> 'model_arch_library',
> 'automl_cifar10_mst_table',
> ARRAY[1,2,3],
> $${'loss': ['categorical_crossentropy'],
> 'optimizer_params_list': [
> {'optimizer': ['Adam'],'lr': [0.0001, 0.01, 'log']},
> {'optimizer': ['RMSprop'],'lr': [0.0001, 0.01, 'log'], 'decay': [1.0e-7, 1.0e-5, 'log']},
> {'optimizer': ['SGD'],'lr': [0.0001, 0.01, 'log'], 'momentum': [0.9, 0.99,'log_near_one']}],
> 'metrics':['accuracy']}$$,
> $${'batch_size': [64,128], 'epochs': [1]}$$,
> 'hyperband',
> 'R=27, eta=3, skip_last=1',
> NULL, -- random state
> NULL, -- object table
> FALSE, -- use GPUs
> 'cifar10_val_packed', -- validation table
> 3, -- metrics compute freq
> NULL, -- name
> NULL); -- descr
> {code}
> produces
> {code}
> select mst_key, metrics_iters, metrics_elapsed_time from hb_cifar10_output_info_r27e3skip1run1 order by mst_key, metrics_iters;
> mst_key | metrics_iters | metrics_elapsed_time
> ---------+---------------+----------------------------------------------------------------------------------------
> 1 | {1} | {5639.45646381378}
> 2 | {1,4} | {5499.1161878109,10417.6853170395}
> 3 | {1} | {5647.45680689812}
> 4 | {1,4} | {5581.32227087021,10422.9603590965}
> 5 | {1} | {5473.09724283218}
> 6 | {1} | {5537.51781392097}
> 7 | {1} | {5468.15699982643}
> 8 | {1} | {5597.43572378159}
> 9 | {1,4} | {5618.64315199852,10291.4588999748}
> 10 | {1} | {5450.7096657753}
> 11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
> 12 | {1} | {5674.29211378098}
> 13 | {1} | {5542.49673891068}
> 14 | {1} | {5682.2088098526}
> 15 | {1,4,7,10,13} | {5610.54115080833,10365.9494991302,7378.03450798988,14876.2571978569,22414.0461809635}
> 16 | {1} | {5563.48544478416}
> 17 | {1,4,7,10,13} | {5661.10293698311,10398.6006779671,7373.00771188736,14871.3066940308,22409.0431649685}
> 18 | {1} | {5626.78051185608}
> 19 | {1} | {5656.17028284073}
> 20 | {1} | {5524.58638978004}
> 21 | {1} | {5555.59004878998}
> 22 | {1} | {5605.62219786644}
> 23 | {1} | {5494.16033291817}
> 24 | {1,4} | {5512.02432179451,10360.8255741596}
> 25 | {1,4} | {5455.62597084045,10296.5835280418}
> 26 | {1,4} | {5589.49808478355,10337.8003959656}
> 27 | {1} | {5576.40678977966}
> 28 | {4,7,10,13} | {10393.5381979942,7413.2452878952,14912.529984951,22449.6516678333}
> 29 | {4} | {10319.5200190544}
> 30 | {4} | {10451.2104020119}
> 31 | {4,7,10,13} | {10328.5663330555,7399.8391699791,14899.122522831,22436.2282240391}
> 32 | {4} | {10437.2220571041}
> 33 | {4} | {10379.5259420872}
> 34 | {4} | {10346.9251220226}
> 35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
> 36 | {4} | {10412.5241000652}
> 37 | {7,10,13} | {7421.8063390255,14921.1340680122,22459.3360388279}
> 38 | {7,10,13} | {7469.96136283875,14970.6747159958,22509.476047039}
> 39 | {7,10,13} | {7461.39016890526,14962.0603349209,22500.7105178833}
> 40 | {7,10,13} | {7435.0908639431,14935.0127699375,22472.9902210236}
> 41 | {7,10,13} | {7391.56045103073,14890.2629489899,22427.4269089699}
> 42 | {7,10,13} | {7354.62691497803,14852.9146559238,22389.92344594}
> (42 rows)
> {code}
> Note that for times with 7 cumulative time is wrong, here are 2 exs:
> {code}
> 11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
> and
> 35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
> {code}
> The expected schedule is:
> {code}
> DROP TABLE IF EXISTS hb_schedule;
> SELECT madlib.hyperband_schedule ('hb_schedule',
> 27,
> 3,
> 1);
> SELECT * FROM hb_schedule ORDER BY s DESC, i;
> s | i | n_i | r_i
> ---+---+-----+-----
> 3 | 0 | 27 | 1
> 3 | 1 | 9 | 3
> 3 | 2 | 3 | 9
> 2 | 0 | 9 | 3
> 2 | 1 | 3 | 9
> 1 | 0 | 6 | 9
> (6 rows)
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)