You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2017/11/30 14:51:13 UTC
[GitHub] marcoabreu opened a new issue #8892: Flaky test_loss.test_ctc_loss_train @ Python3: MKLML-CPU
marcoabreu opened a new issue #8892: Flaky test_loss.test_ctc_loss_train @ Python3: MKLML-CPU
URL: https://github.com/apache/incubator-mxnet/issues/8892
```
======================================================================
FAIL: test_loss.test_ctc_loss_train
----------------------------------------------------------------------
Traceback (most recent call last):
File "/usr/local/lib/python3.5/dist-packages/nose/case.py", line 198, in runTest
self.test(*self.arg)
File "/workspace/tests/python/unittest/test_loss.py", line 204, in test_ctc_loss_train
assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 10
AssertionError:
-------------------- >> begin captured logging << --------------------
root: INFO: Epoch[0] Train-loss=11313.475629
root: INFO: Epoch[0] Time cost=0.030
root: INFO: Epoch[1] Train-loss=189.882272
root: INFO: Epoch[1] Time cost=0.026
root: INFO: Epoch[2] Train-loss=8.424159
root: INFO: Epoch[2] Time cost=0.017
root: INFO: Epoch[3] Train-loss=7.715954
root: INFO: Epoch[3] Time cost=0.007
root: INFO: Epoch[4] Train-loss=8.146382
root: INFO: Epoch[4] Time cost=0.008
root: INFO: Epoch[5] Train-loss=7.868012
root: INFO: Epoch[5] Time cost=0.021
root: INFO: Epoch[6] Train-loss=9.193574
root: INFO: Epoch[6] Time cost=0.198
root: INFO: Epoch[7] Train-loss=7.808158
root: INFO: Epoch[7] Time cost=0.008
root: INFO: Epoch[8] Train-loss=8.120917
root: INFO: Epoch[8] Time cost=0.016
root: INFO: Epoch[9] Train-loss=7.675984
root: INFO: Epoch[9] Time cost=0.008
root: INFO: Epoch[10] Train-loss=7.731975
root: INFO: Epoch[10] Time cost=0.008
root: INFO: Epoch[11] Train-loss=7.634333
root: INFO: Epoch[11] Time cost=0.007
root: INFO: Epoch[12] Train-loss=7.532941
root: INFO: Epoch[12] Time cost=0.007
root: INFO: Epoch[13] Train-loss=7.561222
root: INFO: Epoch[13] Time cost=0.007
root: INFO: Epoch[14] Train-loss=7.597162
root: INFO: Epoch[14] Time cost=0.093
root: INFO: Epoch[15] Train-loss=7.541968
root: INFO: Epoch[15] Time cost=0.144
root: INFO: Epoch[16] Train-loss=7.577588
root: INFO: Epoch[16] Time cost=0.071
root: INFO: Epoch[17] Train-loss=7.509282
root: INFO: Epoch[17] Time cost=0.004
root: INFO: Epoch[18] Train-loss=7.567620
root: INFO: Epoch[18] Time cost=0.004
root: INFO: Epoch[19] Train-loss=7.641724
root: INFO: Epoch[19] Time cost=0.004
root: INFO: Epoch[20] Train-loss=7.614571
root: INFO: Epoch[20] Time cost=0.248
root: INFO: Epoch[21] Train-loss=7.582521
root: INFO: Epoch[21] Time cost=0.229
root: INFO: Epoch[22] Train-loss=7.574559
root: INFO: Epoch[22] Time cost=0.090
root: INFO: Epoch[23] Train-loss=7.592511
root: INFO: Epoch[23] Time cost=0.034
root: INFO: Epoch[24] Train-loss=7.614215
root: INFO: Epoch[24] Time cost=0.009
root: INFO: Epoch[25] Train-loss=7.621218
root: INFO: Epoch[25] Time cost=0.009
root: INFO: Epoch[26] Train-loss=7.616917
root: INFO: Epoch[26] Time cost=0.009
root: INFO: Epoch[27] Train-loss=7.612053
root: INFO: Epoch[27] Time cost=0.009
root: INFO: Epoch[28] Train-loss=7.606215
root: INFO: Epoch[28] Time cost=0.009
root: INFO: Epoch[29] Train-loss=7.595469
root: INFO: Epoch[29] Time cost=0.006
root: INFO: Epoch[30] Train-loss=7.588731
root: INFO: Epoch[30] Time cost=0.006
root: INFO: Epoch[31] Train-loss=7.591382
root: INFO: Epoch[31] Time cost=0.007
root: INFO: Epoch[32] Train-loss=7.599283
root: INFO: Epoch[32] Time cost=0.009
root: INFO: Epoch[33] Train-loss=7.606385
root: INFO: Epoch[33] Time cost=0.007
root: INFO: Epoch[34] Train-loss=7.608895
root: INFO: Epoch[34] Time cost=0.009
root: INFO: Epoch[35] Train-loss=7.607032
root: INFO: Epoch[35] Time cost=0.246
root: INFO: Epoch[36] Train-loss=7.602818
root: INFO: Epoch[36] Time cost=0.005
root: INFO: Epoch[37] Train-loss=7.598604
root: INFO: Epoch[37] Time cost=0.004
root: INFO: Epoch[38] Train-loss=7.596247
root: INFO: Epoch[38] Time cost=0.785
root: INFO: Epoch[39] Train-loss=7.596983
root: INFO: Epoch[39] Time cost=0.256
root: INFO: Epoch[40] Train-loss=7.599925
root: INFO: Epoch[40] Time cost=0.005
root: INFO: Epoch[41] Train-loss=7.603125
root: INFO: Epoch[41] Time cost=0.010
root: INFO: Epoch[42] Train-loss=7.605066
root: INFO: Epoch[42] Time cost=0.005
root: INFO: Epoch[43] Train-loss=7.604844
root: INFO: Epoch[43] Time cost=0.004
root: INFO: Epoch[44] Train-loss=7.602832
root: INFO: Epoch[44] Time cost=0.004
root: INFO: Epoch[45] Train-loss=7.600608
root: INFO: Epoch[45] Time cost=0.007
root: INFO: Epoch[46] Train-loss=7.599727
root: INFO: Epoch[46] Time cost=0.008
root: INFO: Epoch[47] Train-loss=7.600405
root: INFO: Epoch[47] Time cost=0.006
root: INFO: Epoch[48] Train-loss=7.601868
root: INFO: Epoch[48] Time cost=0.006
root: INFO: Epoch[49] Train-loss=7.603262
root: INFO: Epoch[49] Time cost=0.004
root: INFO: Epoch[50] Train-loss=7.604079
root: INFO: Epoch[50] Time cost=0.004
root: INFO: Epoch[51] Train-loss=7.604002
root: INFO: Epoch[51] Time cost=0.025
root: INFO: Epoch[52] Train-loss=7.603129
root: INFO: Epoch[52] Time cost=0.305
root: INFO: Epoch[53] Train-loss=7.602171
root: INFO: Epoch[53] Time cost=0.030
root: INFO: Epoch[54] Train-loss=7.601915
root: INFO: Epoch[54] Time cost=0.016
root: INFO: Epoch[55] Train-loss=7.602428
root: INFO: Epoch[55] Time cost=0.012
root: INFO: Epoch[56] Train-loss=7.603212
root: INFO: Epoch[56] Time cost=0.010
root: INFO: Epoch[57] Train-loss=7.603828
root: INFO: Epoch[57] Time cost=0.036
root: INFO: Epoch[58] Train-loss=7.604122
root: INFO: Epoch[58] Time cost=0.007
root: INFO: Epoch[59] Train-loss=7.604047
root: INFO: Epoch[59] Time cost=0.084
root: INFO: Epoch[60] Train-loss=7.603678
root: INFO: Epoch[60] Time cost=0.004
root: INFO: Epoch[61] Train-loss=7.603335
root: INFO: Epoch[61] Time cost=0.005
root: INFO: Epoch[62] Train-loss=7.603346
root: INFO: Epoch[62] Time cost=0.008
root: INFO: Epoch[63] Train-loss=7.603704
root: INFO: Epoch[63] Time cost=0.008
root: INFO: Epoch[64] Train-loss=7.604126
root: INFO: Epoch[64] Time cost=0.006
root: INFO: Epoch[65] Train-loss=7.604406
root: INFO: Epoch[65] Time cost=0.005
root: INFO: Epoch[66] Train-loss=7.604504
root: INFO: Epoch[66] Time cost=0.005
root: INFO: Epoch[67] Train-loss=7.604446
root: INFO: Epoch[67] Time cost=0.014
root: INFO: Epoch[68] Train-loss=7.604308
root: INFO: Epoch[68] Time cost=0.018
root: INFO: Epoch[69] Train-loss=7.604240
root: INFO: Epoch[69] Time cost=0.018
root: INFO: Epoch[70] Train-loss=7.604355
root: INFO: Epoch[70] Time cost=0.004
root: INFO: Epoch[71] Train-loss=7.604591
root: INFO: Epoch[71] Time cost=0.004
root: INFO: Epoch[72] Train-loss=7.604807
root: INFO: Epoch[72] Time cost=0.004
root: INFO: Epoch[73] Train-loss=7.604928
root: INFO: Epoch[73] Time cost=0.004
root: INFO: Epoch[74] Train-loss=7.604969
root: INFO: Epoch[74] Time cost=0.006
root: INFO: Epoch[75] Train-loss=7.604958
root: INFO: Epoch[75] Time cost=0.007
root: INFO: Epoch[76] Train-loss=7.604938
root: INFO: Epoch[76] Time cost=0.719
root: INFO: Epoch[77] Train-loss=7.604977
root: INFO: Epoch[77] Time cost=0.081
root: INFO: Epoch[78] Train-loss=7.605096
root: INFO: Epoch[78] Time cost=0.023
root: INFO: Epoch[79] Train-loss=7.605244
root: INFO: Epoch[79] Time cost=0.237
root: INFO: Epoch[80] Train-loss=7.605355
root: INFO: Epoch[80] Time cost=0.048
root: INFO: Epoch[81] Train-loss=7.605423
root: INFO: Epoch[81] Time cost=0.057
root: INFO: Epoch[82] Train-loss=7.605456
root: INFO: Epoch[82] Time cost=0.038
root: INFO: Epoch[83] Train-loss=7.605475
root: INFO: Epoch[83] Time cost=0.069
root: INFO: Epoch[84] Train-loss=7.605508
root: INFO: Epoch[84] Time cost=0.061
root: INFO: Epoch[85] Train-loss=7.605579
root: INFO: Epoch[85] Time cost=0.128
root: INFO: Epoch[86] Train-loss=7.605675
root: INFO: Epoch[86] Time cost=0.042
root: INFO: Epoch[87] Train-loss=7.605766
root: INFO: Epoch[87] Time cost=0.007
root: INFO: Epoch[88] Train-loss=7.605834
root: INFO: Epoch[88] Time cost=0.004
root: INFO: Epoch[89] Train-loss=7.605881
root: INFO: Epoch[89] Time cost=0.006
root: INFO: Epoch[90] Train-loss=7.605920
root: INFO: Epoch[90] Time cost=0.010
root: INFO: Epoch[91] Train-loss=7.605960
root: INFO: Epoch[91] Time cost=0.008
root: INFO: Epoch[92] Train-loss=7.606010
root: INFO: Epoch[92] Time cost=0.042
root: INFO: Epoch[93] Train-loss=7.606078
root: INFO: Epoch[93] Time cost=0.008
root: INFO: Epoch[94] Train-loss=7.606150
root: INFO: Epoch[94] Time cost=0.005
root: INFO: Epoch[95] Train-loss=7.606211
root: INFO: Epoch[95] Time cost=0.005
root: INFO: Epoch[96] Train-loss=7.606261
root: INFO: Epoch[96] Time cost=0.031
root: INFO: Epoch[97] Train-loss=7.606306
root: INFO: Epoch[97] Time cost=0.041
root: INFO: Epoch[98] Train-loss=7.606348
root: INFO: Epoch[98] Time cost=0.006
root: INFO: Epoch[99] Train-loss=7.606393
root: INFO: Epoch[99] Time cost=0.061
root: INFO: Epoch[100] Train-loss=7.606446
root: INFO: Epoch[100] Time cost=0.049
root: INFO: Epoch[101] Train-loss=7.606503
root: INFO: Epoch[101] Time cost=0.131
root: INFO: Epoch[102] Train-loss=7.606557
root: INFO: Epoch[102] Time cost=0.547
root: INFO: Epoch[103] Train-loss=7.606606
root: INFO: Epoch[103] Time cost=0.016
root: INFO: Epoch[104] Train-loss=7.606649
root: INFO: Epoch[104] Time cost=0.008
root: INFO: Epoch[105] Train-loss=7.606690
root: INFO: Epoch[105] Time cost=0.008
root: INFO: Epoch[106] Train-loss=7.606733
root: INFO: Epoch[106] Time cost=0.007
root: INFO: Epoch[107] Train-loss=7.606778
root: INFO: Epoch[107] Time cost=0.008
root: INFO: Epoch[108] Train-loss=7.606827
root: INFO: Epoch[108] Time cost=0.019
root: INFO: Epoch[109] Train-loss=7.606872
root: INFO: Epoch[109] Time cost=0.033
root: INFO: Epoch[110] Train-loss=7.606918
root: INFO: Epoch[110] Time cost=0.009
root: INFO: Epoch[111] Train-loss=7.606959
root: INFO: Epoch[111] Time cost=0.007
root: INFO: Epoch[112] Train-loss=7.606998
root: INFO: Epoch[112] Time cost=0.009
root: INFO: Epoch[113] Train-loss=7.607038
root: INFO: Epoch[113] Time cost=0.011
root: INFO: Epoch[114] Train-loss=7.607080
root: INFO: Epoch[114] Time cost=0.004
root: INFO: Epoch[115] Train-loss=7.607121
root: INFO: Epoch[115] Time cost=0.004
root: INFO: Epoch[116] Train-loss=7.607162
root: INFO: Epoch[116] Time cost=0.004
root: INFO: Epoch[117] Train-loss=7.607202
root: INFO: Epoch[117] Time cost=0.004
root: INFO: Epoch[118] Train-loss=7.607243
root: INFO: Epoch[118] Time cost=0.004
root: INFO: Epoch[119] Train-loss=7.607280
root: INFO: Epoch[119] Time cost=0.004
root: INFO: Epoch[120] Train-loss=7.607315
root: INFO: Epoch[120] Time cost=0.004
root: INFO: Epoch[121] Train-loss=7.607354
root: INFO: Epoch[121] Time cost=0.005
root: INFO: Epoch[122] Train-loss=7.607391
root: INFO: Epoch[122] Time cost=0.004
root: INFO: Epoch[123] Train-loss=7.607427
root: INFO: Epoch[123] Time cost=0.004
root: INFO: Epoch[124] Train-loss=7.607465
root: INFO: Epoch[124] Time cost=0.033
root: INFO: Epoch[125] Train-loss=7.607501
root: INFO: Epoch[125] Time cost=0.050
root: INFO: Epoch[126] Train-loss=7.607535
root: INFO: Epoch[126] Time cost=0.050
root: INFO: Epoch[127] Train-loss=7.607567
root: INFO: Epoch[127] Time cost=0.016
root: INFO: Epoch[128] Train-loss=7.607602
root: INFO: Epoch[128] Time cost=0.008
root: INFO: Epoch[129] Train-loss=7.607638
root: INFO: Epoch[129] Time cost=0.012
root: INFO: Epoch[130] Train-loss=7.607672
root: INFO: Epoch[130] Time cost=0.022
root: INFO: Epoch[131] Train-loss=7.607704
root: INFO: Epoch[131] Time cost=0.675
root: INFO: Epoch[132] Train-loss=7.607737
root: INFO: Epoch[132] Time cost=0.173
root: INFO: Epoch[133] Train-loss=7.607770
root: INFO: Epoch[133] Time cost=0.005
root: INFO: Epoch[134] Train-loss=7.607801
root: INFO: Epoch[134] Time cost=0.007
root: INFO: Epoch[135] Train-loss=7.607832
root: INFO: Epoch[135] Time cost=0.095
root: INFO: Epoch[136] Train-loss=7.607864
root: INFO: Epoch[136] Time cost=0.006
root: INFO: Epoch[137] Train-loss=7.607893
root: INFO: Epoch[137] Time cost=0.005
root: INFO: Epoch[138] Train-loss=7.607925
root: INFO: Epoch[138] Time cost=0.005
root: INFO: Epoch[139] Train-loss=7.607956
root: INFO: Epoch[139] Time cost=0.007
root: INFO: Epoch[140] Train-loss=7.607986
root: INFO: Epoch[140] Time cost=0.005
root: INFO: Epoch[141] Train-loss=7.608013
root: INFO: Epoch[141] Time cost=0.005
root: INFO: Epoch[142] Train-loss=7.608044
root: INFO: Epoch[142] Time cost=0.004
root: INFO: Epoch[143] Train-loss=7.608073
root: INFO: Epoch[143] Time cost=0.008
root: INFO: Epoch[144] Train-loss=7.608100
root: INFO: Epoch[144] Time cost=0.004
root: INFO: Epoch[145] Train-loss=7.608127
root: INFO: Epoch[145] Time cost=0.005
root: INFO: Epoch[146] Train-loss=7.608156
root: INFO: Epoch[146] Time cost=0.006
root: INFO: Epoch[147] Train-loss=7.608185
root: INFO: Epoch[147] Time cost=0.004
root: INFO: Epoch[148] Train-loss=7.608209
root: INFO: Epoch[148] Time cost=0.004
root: INFO: Epoch[149] Train-loss=7.608239
root: INFO: Epoch[149] Time cost=0.005
root: INFO: Epoch[150] Train-loss=7.608265
root: INFO: Epoch[150] Time cost=0.048
root: INFO: Epoch[151] Train-loss=7.608290
root: INFO: Epoch[151] Time cost=0.042
root: INFO: Epoch[152] Train-loss=7.608318
root: INFO: Epoch[152] Time cost=0.007
root: INFO: Epoch[153] Train-loss=7.608344
root: INFO: Epoch[153] Time cost=0.362
root: INFO: Epoch[154] Train-loss=7.608369
root: INFO: Epoch[154] Time cost=0.008
root: INFO: Epoch[155] Train-loss=7.608394
root: INFO: Epoch[155] Time cost=0.007
root: INFO: Epoch[156] Train-loss=7.608420
root: INFO: Epoch[156] Time cost=0.010
root: INFO: Epoch[157] Train-loss=7.608445
root: INFO: Epoch[157] Time cost=0.027
root: INFO: Epoch[158] Train-loss=7.608467
root: INFO: Epoch[158] Time cost=0.197
root: INFO: Epoch[159] Train-loss=7.608494
root: INFO: Epoch[159] Time cost=0.018
root: INFO: Epoch[160] Train-loss=7.608517
root: INFO: Epoch[160] Time cost=0.265
root: INFO: Epoch[161] Train-loss=7.608541
root: INFO: Epoch[161] Time cost=0.450
root: INFO: Epoch[162] Train-loss=7.608565
root: INFO: Epoch[162] Time cost=0.063
root: INFO: Epoch[163] Train-loss=7.608587
root: INFO: Epoch[163] Time cost=0.013
root: INFO: Epoch[164] Train-loss=7.608611
root: INFO: Epoch[164] Time cost=0.093
root: INFO: Epoch[165] Train-loss=7.608633
root: INFO: Epoch[165] Time cost=0.009
root: INFO: Epoch[166] Train-loss=7.608657
root: INFO: Epoch[166] Time cost=0.004
root: INFO: Epoch[167] Train-loss=7.608584
root: INFO: Epoch[167] Time cost=0.010
root: INFO: Epoch[168] Train-loss=7.607704
root: INFO: Epoch[168] Time cost=0.019
root: INFO: Epoch[169] Train-loss=7.611019
root: INFO: Epoch[169] Time cost=0.005
root: INFO: Epoch[170] Train-loss=7.614492
root: INFO: Epoch[170] Time cost=0.004
root: INFO: Epoch[171] Train-loss=7.619690
root: INFO: Epoch[171] Time cost=0.005
root: INFO: Epoch[172] Train-loss=7.614747
root: INFO: Epoch[172] Time cost=0.004
root: INFO: Epoch[173] Train-loss=7.610207
root: INFO: Epoch[173] Time cost=0.004
root: INFO: Epoch[174] Train-loss=7.617975
root: INFO: Epoch[174] Time cost=0.004
root: INFO: Epoch[175] Train-loss=7.613567
root: INFO: Epoch[175] Time cost=0.004
root: INFO: Epoch[176] Train-loss=7.607732
root: INFO: Epoch[176] Time cost=0.004
root: INFO: Epoch[177] Train-loss=7.613142
root: INFO: Epoch[177] Time cost=0.004
root: INFO: Epoch[178] Train-loss=7.618055
root: INFO: Epoch[178] Time cost=0.020
root: INFO: Epoch[179] Train-loss=7.595630
root: INFO: Epoch[179] Time cost=0.022
root: INFO: Epoch[180] Train-loss=7.597824
root: INFO: Epoch[180] Time cost=0.021
root: INFO: Epoch[181] Train-loss=7.629328
root: INFO: Epoch[181] Time cost=0.052
root: INFO: Epoch[182] Train-loss=7.617379
root: INFO: Epoch[182] Time cost=0.006
root: INFO: Epoch[183] Train-loss=7.558949
root: INFO: Epoch[183] Time cost=0.004
root: INFO: Epoch[184] Train-loss=7.601670
root: INFO: Epoch[184] Time cost=0.006
root: INFO: Epoch[185] Train-loss=7.580250
root: INFO: Epoch[185] Time cost=0.017
root: INFO: Epoch[186] Train-loss=7.557942
root: INFO: Epoch[186] Time cost=0.004
root: INFO: Epoch[187] Train-loss=7.549942
root: INFO: Epoch[187] Time cost=0.005
root: INFO: Epoch[188] Train-loss=7.565051
root: INFO: Epoch[188] Time cost=0.175
root: INFO: Epoch[189] Train-loss=7.554461
root: INFO: Epoch[189] Time cost=0.539
root: INFO: Epoch[190] Train-loss=7.506800
root: INFO: Epoch[190] Time cost=0.146
root: INFO: Epoch[191] Train-loss=7.549119
root: INFO: Epoch[191] Time cost=0.007
root: INFO: Epoch[192] Train-loss=7.636117
root: INFO: Epoch[192] Time cost=0.007
root: INFO: Epoch[193] Train-loss=7.616981
root: INFO: Epoch[193] Time cost=0.073
root: INFO: Epoch[194] Train-loss=7.683337
root: INFO: Epoch[194] Time cost=0.006
root: INFO: Epoch[195] Train-loss=14.166114
root: INFO: Epoch[195] Time cost=0.005
root: INFO: Epoch[196] Train-loss=9.231956
root: INFO: Epoch[196] Time cost=0.006
root: INFO: Epoch[197] Train-loss=14.450128
root: INFO: Epoch[197] Time cost=0.023
root: INFO: Epoch[198] Train-loss=3422.130003
root: INFO: Epoch[198] Time cost=0.049
root: INFO: Epoch[199] Train-loss=16.536609
root: INFO: Epoch[199] Time cost=0.016
--------------------- >> end captured logging << ---------------------
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services