You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@madlib.apache.org by "Frank McQuillan (Jira)" <ji...@apache.org> on 2020/10/12 23:05:00 UTC
[jira] [Created] (MADLIB-1456) Hyperband cumulative output time is
not correct
Frank McQuillan created MADLIB-1456:
---------------------------------------
Summary: Hyperband cumulative output time is not correct
Key: MADLIB-1456
URL: https://issues.apache.org/jira/browse/MADLIB-1456
Project: Apache MADlib
Issue Type: Bug
Components: Deep Learning
Reporter: Frank McQuillan
Fix For: v1.18.0
{code}
SELECT madlib.madlib_keras_automl('cifar10_train_packed',
'automl_cifar10_output',
'model_arch_library',
'automl_cifar10_mst_table',
ARRAY[1,2,3],
$${'loss': ['categorical_crossentropy'],
'optimizer_params_list': [
{'optimizer': ['Adam'],'lr': [0.0001, 0.01, 'log']},
{'optimizer': ['RMSprop'],'lr': [0.0001, 0.01, 'log'], 'decay': [1.0e-7, 1.0e-5, 'log']},
{'optimizer': ['SGD'],'lr': [0.0001, 0.01, 'log'], 'momentum': [0.9, 0.99,'log_near_one']}],
'metrics':['accuracy']}$$,
$${'batch_size': [64,128], 'epochs': [1]}$$,
'hyperband',
'R=27, eta=3, skip_last=1',
NULL, -- random state
NULL, -- object table
FALSE, -- use GPUs
'cifar10_val_packed', -- validation table
3, -- metrics compute freq
NULL, -- name
NULL); -- descr
{code}
produces
{code}
select mst_key, metrics_iters, metrics_elapsed_time from hb_cifar10_output_info_r27e3skip1run1 order by mst_key, metrics_iters;
mst_key | metrics_iters | metrics_elapsed_time
---------+---------------+----------------------------------------------------------------------------------------
1 | {1} | {5639.45646381378}
2 | {1,4} | {5499.1161878109,10417.6853170395}
3 | {1} | {5647.45680689812}
4 | {1,4} | {5581.32227087021,10422.9603590965}
5 | {1} | {5473.09724283218}
6 | {1} | {5537.51781392097}
7 | {1} | {5468.15699982643}
8 | {1} | {5597.43572378159}
9 | {1,4} | {5618.64315199852,10291.4588999748}
10 | {1} | {5450.7096657753}
11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
12 | {1} | {5674.29211378098}
13 | {1} | {5542.49673891068}
14 | {1} | {5682.2088098526}
15 | {1,4,7,10,13} | {5610.54115080833,10365.9494991302,7378.03450798988,14876.2571978569,22414.0461809635}
16 | {1} | {5563.48544478416}
17 | {1,4,7,10,13} | {5661.10293698311,10398.6006779671,7373.00771188736,14871.3066940308,22409.0431649685}
18 | {1} | {5626.78051185608}
19 | {1} | {5656.17028284073}
20 | {1} | {5524.58638978004}
21 | {1} | {5555.59004878998}
22 | {1} | {5605.62219786644}
23 | {1} | {5494.16033291817}
24 | {1,4} | {5512.02432179451,10360.8255741596}
25 | {1,4} | {5455.62597084045,10296.5835280418}
26 | {1,4} | {5589.49808478355,10337.8003959656}
27 | {1} | {5576.40678977966}
28 | {4,7,10,13} | {10393.5381979942,7413.2452878952,14912.529984951,22449.6516678333}
29 | {4} | {10319.5200190544}
30 | {4} | {10451.2104020119}
31 | {4,7,10,13} | {10328.5663330555,7399.8391699791,14899.122522831,22436.2282240391}
32 | {4} | {10437.2220571041}
33 | {4} | {10379.5259420872}
34 | {4} | {10346.9251220226}
35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
36 | {4} | {10412.5241000652}
37 | {7,10,13} | {7421.8063390255,14921.1340680122,22459.3360388279}
38 | {7,10,13} | {7469.96136283875,14970.6747159958,22509.476047039}
39 | {7,10,13} | {7461.39016890526,14962.0603349209,22500.7105178833}
40 | {7,10,13} | {7435.0908639431,14935.0127699375,22472.9902210236}
41 | {7,10,13} | {7391.56045103073,14890.2629489899,22427.4269089699}
42 | {7,10,13} | {7354.62691497803,14852.9146559238,22389.92344594}
(42 rows)
{code}
Note that for times with 7 cumulative time is wrong
{code}
11 | {1,4,7,10,13} | {5485.84163188934,10282.08061409,7448.4407389164,14948.3819220066,22486.4891610146}
and
35 | {4,7,10,13} | {10310.3271739483,7368.01132202148,14866.3002450466,22403.3542098999}
{code}
The expected schedule is:
{code}
DROP TABLE IF EXISTS hb_schedule;
SELECT madlib.hyperband_schedule ('hb_schedule',
27,
3,
1);
SELECT * FROM hb_schedule ORDER BY s DESC, i;
s | i | n_i | r_i
---+---+-----+-----
3 | 0 | 27 | 1
3 | 1 | 9 | 3
3 | 2 | 3 | 9
2 | 0 | 9 | 3
2 | 1 | 3 | 9
1 | 0 | 6 | 9
(6 rows)
{code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)