You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@mxnet.apache.org by GitBox <gi...@apache.org> on 2022/06/16 06:58:43 UTC
[GitHub] [incubator-mxnet] jaeriver commented on issue #21069: Fail to load NLP Bert Model in AWS Lambda Arm64 (Graviton 2)
jaeriver commented on issue #21069:
URL: https://github.com/apache/incubator-mxnet/issues/21069#issuecomment-1157304981
### Bert model info (model-symbol.json)
```
{
"nodes": [
{
"op": "null",
"name": "data0",
"inputs": []
},
{
"op": "null",
"name": "bertmodel0_word_embed_embedding0_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(30522, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Embedding",
"name": "bertmodel0_word_embed_embedding0_fwd",
"attrs": {
"dtype": "float32",
"input_dim": "30522",
"output_dim": "768",
"sparse_grad": "False"
},
"inputs": [[0, 0, 0], [1, 0, 0]]
},
{
"op": "null",
"name": "data1",
"inputs": []
},
{
"op": "null",
"name": "bertmodel0_token_type_embed_embedding0_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(2, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Embedding",
"name": "bertmodel0_token_type_embed_embedding0_fwd",
"attrs": {
"dtype": "float32",
"input_dim": "2",
"output_dim": "768",
"sparse_grad": "False"
},
"inputs": [[3, 0, 0], [4, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertmodel0__plus0",
"inputs": [[2, 0, 0], [5, 0, 0]]
},
{
"op": "transpose",
"name": "bertmodel0_transpose0",
"attrs": {"axes": "(1, 0, 2)"},
"inputs": [[6, 0, 0]]
},
{
"op": "_contrib_arange_like",
"name": "bertencoder0_arange_like0",
"attrs": {"axis": "0"},
"inputs": [[7, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_position_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(512, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Embedding",
"name": "bertencoder0_embedding0",
"attrs": {
"input_dim": "512",
"output_dim": "768"
},
"inputs": [[8, 0, 0], [9, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[10, 0, 0]]
},
{
"op": "broadcast_add",
"name": "bertencoder0_broadcast_add1",
"inputs": [[7, 0, 0], [11, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[12, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[13, 0, 0], [14, 0, 0], [15, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[17, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[19, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[21, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[18, 0, 0], [20, 0, 0], [22, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[23, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[25, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[27, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[29, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[26, 0, 0], [28, 0, 0], [30, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[31, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[16, 0, 0], [24, 0, 0], [32, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[33, 0, 0]]
},
{
"op": "null",
"name": "data2",
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_reshape0",
"attrs": {"shape": "(-1, 1)"},
"inputs": [[35, 0, 0]]
},
{
"op": "zeros_like",
"name": "bertencoder0_zeros_like0",
"inputs": [[8, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_reshape1",
"attrs": {"shape": "(1, -1)"},
"inputs": [[37, 0, 0]]
},
{
"op": "broadcast_add",
"name": "bertencoder0_broadcast_add0",
"inputs": [[36, 0, 0], [38, 0, 0]]
},
{
"op": "Cast",
"name": "bertencoder0_cast0",
"attrs": {"dtype": "int32"},
"inputs": [[39, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[41, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[42, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[34, 0, 0], [43, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[44, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer0_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[33, 0, 0], [45, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer0_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer0_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[46, 0, 0], [47, 0, 0], [48, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[49, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer0__plus0",
"inputs": [[50, 0, 0], [16, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[51, 0, 0], [52, 0, 0], [53, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer0_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer0_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[54, 0, 0], [55, 0, 0], [56, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer0_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[57, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer0_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer0_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[58, 0, 0], [59, 0, 0], [60, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer0_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[61, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer0_positionwiseffn0__plus0",
"inputs": [[62, 0, 0], [54, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer0_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer0_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer0_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[63, 0, 0], [64, 0, 0], [65, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[67, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[69, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[71, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[68, 0, 0], [70, 0, 0], [72, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[73, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[75, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[77, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[79, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[76, 0, 0], [78, 0, 0], [80, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[81, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[66, 0, 0], [74, 0, 0], [82, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[83, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[85, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[86, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[84, 0, 0], [87, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[88, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer1_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[83, 0, 0], [89, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer1_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer1_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[90, 0, 0], [91, 0, 0], [92, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer1_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[93, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer1__plus0",
"inputs": [[94, 0, 0], [66, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer1_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer1_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[95, 0, 0], [96, 0, 0], [97, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer1_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer1_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[98, 0, 0], [99, 0, 0], [100, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer1_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[101, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer1_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer1_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[102, 0, 0], [103, 0, 0], [104, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer1_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[105, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer1_positionwiseffn0__plus0",
"inputs": [[106, 0, 0], [98, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer1_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer1_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer1_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[107, 0, 0], [108, 0, 0], [109, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[111, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[113, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[115, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[112, 0, 0], [114, 0, 0], [116, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[117, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[119, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[121, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[123, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[120, 0, 0], [122, 0, 0], [124, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[125, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[110, 0, 0], [118, 0, 0], [126, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[127, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[129, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[130, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[128, 0, 0], [131, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[132, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer2_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[127, 0, 0], [133, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer2_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer2_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[134, 0, 0], [135, 0, 0], [136, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer2_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[137, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer2__plus0",
"inputs": [[138, 0, 0], [110, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer2_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer2_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[139, 0, 0], [140, 0, 0], [141, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer2_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer2_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[142, 0, 0], [143, 0, 0], [144, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer2_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[145, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer2_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer2_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[146, 0, 0], [147, 0, 0], [148, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer2_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[149, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer2_positionwiseffn0__plus0",
"inputs": [[150, 0, 0], [142, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer2_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer2_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer2_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[151, 0, 0], [152, 0, 0], [153, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[155, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[157, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[159, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[156, 0, 0], [158, 0, 0], [160, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[161, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[163, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[165, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[167, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[164, 0, 0], [166, 0, 0], [168, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[169, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[154, 0, 0], [162, 0, 0], [170, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[171, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[173, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[174, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[172, 0, 0], [175, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[176, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer3_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[171, 0, 0], [177, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer3_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer3_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[178, 0, 0], [179, 0, 0], [180, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer3_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[181, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer3__plus0",
"inputs": [[182, 0, 0], [154, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer3_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer3_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[183, 0, 0], [184, 0, 0], [185, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer3_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer3_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[186, 0, 0], [187, 0, 0], [188, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer3_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[189, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer3_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer3_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[190, 0, 0], [191, 0, 0], [192, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer3_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[193, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer3_positionwiseffn0__plus0",
"inputs": [[194, 0, 0], [186, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer3_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer3_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer3_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[195, 0, 0], [196, 0, 0], [197, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[199, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[201, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[203, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[200, 0, 0], [202, 0, 0], [204, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[205, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[207, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[209, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[211, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[208, 0, 0], [210, 0, 0], [212, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[213, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[198, 0, 0], [206, 0, 0], [214, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[215, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[217, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[218, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[216, 0, 0], [219, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[220, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer4_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[215, 0, 0], [221, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer4_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer4_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[222, 0, 0], [223, 0, 0], [224, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer4_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[225, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer4__plus0",
"inputs": [[226, 0, 0], [198, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer4_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer4_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[227, 0, 0], [228, 0, 0], [229, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer4_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer4_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[230, 0, 0], [231, 0, 0], [232, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer4_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[233, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer4_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer4_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[234, 0, 0], [235, 0, 0], [236, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer4_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[237, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer4_positionwiseffn0__plus0",
"inputs": [[238, 0, 0], [230, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer4_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer4_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer4_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[239, 0, 0], [240, 0, 0], [241, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[243, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[245, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[247, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[244, 0, 0], [246, 0, 0], [248, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[249, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[251, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[253, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[255, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[252, 0, 0], [254, 0, 0], [256, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[257, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[242, 0, 0], [250, 0, 0], [258, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[259, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[261, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[262, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[260, 0, 0], [263, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[264, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer5_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[259, 0, 0], [265, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer5_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer5_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[266, 0, 0], [267, 0, 0], [268, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer5_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[269, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer5__plus0",
"inputs": [[270, 0, 0], [242, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer5_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer5_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[271, 0, 0], [272, 0, 0], [273, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer5_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer5_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[274, 0, 0], [275, 0, 0], [276, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer5_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[277, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer5_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer5_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[278, 0, 0], [279, 0, 0], [280, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer5_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[281, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer5_positionwiseffn0__plus0",
"inputs": [[282, 0, 0], [274, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer5_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer5_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer5_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[283, 0, 0], [284, 0, 0], [285, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[287, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[289, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[291, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[288, 0, 0], [290, 0, 0], [292, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[293, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[295, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[297, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[299, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[296, 0, 0], [298, 0, 0], [300, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[301, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[286, 0, 0], [294, 0, 0], [302, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[303, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[305, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[306, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[304, 0, 0], [307, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[308, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer6_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[303, 0, 0], [309, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer6_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer6_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[310, 0, 0], [311, 0, 0], [312, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer6_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[313, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer6__plus0",
"inputs": [[314, 0, 0], [286, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer6_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer6_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[315, 0, 0], [316, 0, 0], [317, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer6_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer6_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[318, 0, 0], [319, 0, 0], [320, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer6_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[321, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer6_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer6_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[322, 0, 0], [323, 0, 0], [324, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer6_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[325, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer6_positionwiseffn0__plus0",
"inputs": [[326, 0, 0], [318, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer6_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer6_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer6_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[327, 0, 0], [328, 0, 0], [329, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[331, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[333, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[335, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[332, 0, 0], [334, 0, 0], [336, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[337, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[339, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[341, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[343, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[340, 0, 0], [342, 0, 0], [344, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[345, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[330, 0, 0], [338, 0, 0], [346, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[347, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[349, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[350, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[348, 0, 0], [351, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[352, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer7_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[347, 0, 0], [353, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer7_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer7_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[354, 0, 0], [355, 0, 0], [356, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer7_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[357, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer7__plus0",
"inputs": [[358, 0, 0], [330, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer7_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer7_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[359, 0, 0], [360, 0, 0], [361, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer7_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer7_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[362, 0, 0], [363, 0, 0], [364, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer7_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[365, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer7_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer7_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[366, 0, 0], [367, 0, 0], [368, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer7_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[369, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer7_positionwiseffn0__plus0",
"inputs": [[370, 0, 0], [362, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer7_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer7_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer7_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[371, 0, 0], [372, 0, 0], [373, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[375, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[377, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[379, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[376, 0, 0], [378, 0, 0], [380, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[381, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[383, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[385, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[387, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[384, 0, 0], [386, 0, 0], [388, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[389, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[374, 0, 0], [382, 0, 0], [390, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[391, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[393, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[394, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[392, 0, 0], [395, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[396, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer8_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[391, 0, 0], [397, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer8_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer8_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[398, 0, 0], [399, 0, 0], [400, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer8_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[401, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer8__plus0",
"inputs": [[402, 0, 0], [374, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer8_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer8_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[403, 0, 0], [404, 0, 0], [405, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer8_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer8_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[406, 0, 0], [407, 0, 0], [408, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer8_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[409, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer8_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer8_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[410, 0, 0], [411, 0, 0], [412, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer8_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[413, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer8_positionwiseffn0__plus0",
"inputs": [[414, 0, 0], [406, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer8_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer8_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer8_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[415, 0, 0], [416, 0, 0], [417, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[419, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[421, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[423, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[420, 0, 0], [422, 0, 0], [424, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[425, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[427, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[429, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[431, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[428, 0, 0], [430, 0, 0], [432, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[433, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[418, 0, 0], [426, 0, 0], [434, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[435, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[437, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[438, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[436, 0, 0], [439, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[440, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer9_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[435, 0, 0], [441, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer9_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer9_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[442, 0, 0], [443, 0, 0], [444, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer9_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[445, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer9__plus0",
"inputs": [[446, 0, 0], [418, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer9_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer9_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[447, 0, 0], [448, 0, 0], [449, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer9_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer9_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[450, 0, 0], [451, 0, 0], [452, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer9_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[453, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer9_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer9_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[454, 0, 0], [455, 0, 0], [456, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer9_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[457, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer9_positionwiseffn0__plus0",
"inputs": [[458, 0, 0], [450, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer9_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer9_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer9_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[459, 0, 0], [460, 0, 0], [461, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[463, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[465, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[467, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[464, 0, 0], [466, 0, 0], [468, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[469, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[471, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[473, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[475, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[472, 0, 0], [474, 0, 0], [476, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[477, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[462, 0, 0], [470, 0, 0], [478, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[479, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[481, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[482, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[480, 0, 0], [483, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[484, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer10_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[479, 0, 0], [485, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer10_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer10_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[486, 0, 0], [487, 0, 0], [488, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer10_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[489, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer10__plus0",
"inputs": [[490, 0, 0], [462, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer10_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer10_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[491, 0, 0], [492, 0, 0], [493, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer10_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer10_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[494, 0, 0], [495, 0, 0], [496, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer10_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[497, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer10_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer10_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[498, 0, 0], [499, 0, 0], [500, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer10_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[501, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer10_positionwiseffn0__plus0",
"inputs": [[502, 0, 0], [494, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer10_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer10_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer10_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[503, 0, 0], [504, 0, 0], [505, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_query_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape0",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[507, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_key_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape1",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[509, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_value_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape2",
"attrs": {
"reverse": "True",
"shape": "(12, -1, 0)"
},
"inputs": [[511, 0, 0]]
},
{
"op": "Concat",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_concat0",
"attrs": {
"dim": "-2",
"num_args": "3"
},
"inputs": [[508, 0, 0], [510, 0, 0], [512, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape3",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[513, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_query_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape4",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[515, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_key_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape5",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[517, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_value_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape6",
"attrs": {
"reverse": "True",
"shape": "(12, -1)"
},
"inputs": [[519, 0, 0]]
},
{
"op": "stack",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_stack0",
"attrs": {
"axis": "1",
"num_args": "3"
},
"inputs": [[516, 0, 0], [518, 0, 0], [520, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape7",
"attrs": {"shape": "-1"},
"inputs": [[521, 0, 0]]
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_fullyconnected0",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "2304"
},
"inputs": [[506, 0, 0], [514, 0, 0], [522, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_qk",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_interleaved_matmul_selfatt_qk0",
"attrs": {"heads": "12"},
"inputs": [[523, 0, 0]]
},
{
"op": "expand_dims",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_expand_dims0",
"attrs": {"axis": "1"},
"inputs": [[40, 0, 0]]
},
{
"op": "broadcast_axis",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_broadcast_axis0",
"attrs": {
"axis": "1",
"size": "12"
},
"inputs": [[525, 0, 0]]
},
{
"op": "Reshape",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_reshape8",
"attrs": {
"reverse": "True",
"shape": "(-1, 0)"
},
"inputs": [[526, 0, 0]]
},
{
"op": "softmax",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_softmax0",
"attrs": {
"axis": "-1",
"use_length": "True"
},
"inputs": [[524, 0, 0], [527, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[528, 0, 0]]
},
{
"op": "_contrib_interleaved_matmul_selfatt_valatt",
"name": "bertencoder0_transformer11_dotproductselfattentioncell0_interleaved_matmul_selfatt_valatt0",
"attrs": {"heads": "12"},
"inputs": [[523, 0, 0], [529, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_proj_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer11_proj_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer11_proj_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[530, 0, 0], [531, 0, 0], [532, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer11_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[533, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer11__plus0",
"inputs": [[534, 0, 0], [506, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer11_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer11_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[535, 0, 0], [536, 0, 0], [537, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_positionwiseffn0_ffn_1_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(3072, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer11_positionwiseffn0_ffn_1_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(3072,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer11_positionwiseffn0_ffn_1_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "3072"
},
"inputs": [[538, 0, 0], [539, 0, 0], [540, 0, 0]]
},
{
"op": "LeakyReLU",
"name": "bertencoder0_transformer11_positionwiseffn0_gelu0_leakyrelu0",
"attrs": {"act_type": "gelu"},
"inputs": [[541, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_positionwiseffn0_ffn_2_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 3072)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer11_positionwiseffn0_ffn_2_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertencoder0_transformer11_positionwiseffn0_ffn_2_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[542, 0, 0], [543, 0, 0], [544, 0, 0]]
},
{
"op": "Dropout",
"name": "bertencoder0_transformer11_positionwiseffn0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[545, 0, 0]]
},
{
"op": "elemwise_add",
"name": "bertencoder0_transformer11_positionwiseffn0__plus0",
"inputs": [[546, 0, 0], [538, 0, 0]]
},
{
"op": "null",
"name": "bertencoder0_transformer11_positionwiseffn0_layernorm0_gamma",
"attrs": {
"__dtype__": "0",
"__init__": "ones",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertencoder0_transformer11_positionwiseffn0_layernorm0_beta",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "LayerNorm",
"name": "bertencoder0_transformer11_positionwiseffn0_layernorm0_layernorm0",
"attrs": {
"axis": "-1",
"eps": "1e-12"
},
"inputs": [[547, 0, 0], [548, 0, 0], [549, 0, 0]]
},
{
"op": "SequenceMask",
"name": "bertencoder0_sequencemask0",
"attrs": {
"axis": "0",
"use_sequence_length": "True"
},
"inputs": [[550, 0, 0], [35, 0, 0]]
},
{
"op": "transpose",
"name": "bertmodel0_transpose1",
"attrs": {"axes": "(1, 0, 2)"},
"inputs": [[551, 0, 0]]
},
{
"op": "slice",
"name": "bertmodel0_slice0",
"attrs": {
"begin": "(0, 0, 0)",
"end": "(None, 1, None)"
},
"inputs": [[552, 0, 0]]
},
{
"op": "Reshape",
"name": "bertmodel0_reshape0",
"attrs": {"shape": "(-1, 768)"},
"inputs": [[553, 0, 0]]
},
{
"op": "null",
"name": "bertmodel0_pooler_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(768, 768)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertmodel0_pooler_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(768,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertmodel0_pooler_fwd",
"attrs": {
"flatten": "False",
"no_bias": "False",
"num_hidden": "768"
},
"inputs": [[554, 0, 0], [555, 0, 0], [556, 0, 0]]
},
{
"op": "Activation",
"name": "bertmodel0_pooler_tanh_fwd",
"attrs": {"act_type": "tanh"},
"inputs": [[557, 0, 0]]
},
{
"op": "Dropout",
"name": "bertclassifier0_dropout0_fwd",
"attrs": {
"axes": "()",
"cudnn_off": "False",
"p": "0.1"
},
"inputs": [[558, 0, 0]]
},
{
"op": "null",
"name": "bertclassifier0_dense0_weight",
"attrs": {
"__dtype__": "0",
"__lr_mult__": "1.0",
"__shape__": "(2, 0)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "null",
"name": "bertclassifier0_dense0_bias",
"attrs": {
"__dtype__": "0",
"__init__": "zeros",
"__lr_mult__": "1.0",
"__shape__": "(2,)",
"__storage_type__": "0",
"__wd_mult__": "1.0"
},
"inputs": []
},
{
"op": "FullyConnected",
"name": "bertclassifier0_dense0_fwd",
"attrs": {
"flatten": "True",
"no_bias": "False",
"num_hidden": "2"
},
"inputs": [[559, 0, 0], [560, 0, 0], [561, 0, 0]]
}
],
"arg_nodes": [
0,
1,
3,
4,
9,
14,
15,
17,
19,
21,
25,
27,
29,
35,
47,
48,
52,
53,
55,
56,
59,
60,
64,
65,
67,
69,
71,
75,
77,
79,
91,
92,
96,
97,
99,
100,
103,
104,
108,
109,
111,
113,
115,
119,
121,
123,
135,
136,
140,
141,
143,
144,
147,
148,
152,
153,
155,
157,
159,
163,
165,
167,
179,
180,
184,
185,
187,
188,
191,
192,
196,
197,
199,
201,
203,
207,
209,
211,
223,
224,
228,
229,
231,
232,
235,
236,
240,
241,
243,
245,
247,
251,
253,
255,
267,
268,
272,
273,
275,
276,
279,
280,
284,
285,
287,
289,
291,
295,
297,
299,
311,
312,
316,
317,
319,
320,
323,
324,
328,
329,
331,
333,
335,
339,
341,
343,
355,
356,
360,
361,
363,
364,
367,
368,
372,
373,
375,
377,
379,
383,
385,
387,
399,
400,
404,
405,
407,
408,
411,
412,
416,
417,
419,
421,
423,
427,
429,
431,
443,
444,
448,
449,
451,
452,
455,
456,
460,
461,
463,
465,
467,
471,
473,
475,
487,
488,
492,
493,
495,
496,
499,
500,
504,
505,
507,
509,
511,
515,
517,
519,
531,
532,
536,
537,
539,
540,
543,
544,
548,
549,
555,
556,
560,
561
],
"node_row_ptr": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9,
10,
11,
12,
13,
15,
16,
17,
20,
21,
22,
23,
24,
25,
26,
27,
28,
29,
30,
31,
32,
33,
34,
35,
36,
37,
38,
39,
40,
41,
42,
43,
44,
45,
46,
47,
48,
50,
51,
52,
53,
54,
56,
57,
58,
59,
62,
63,
64,
65,
66,
67,
68,
69,
71,
72,
73,
74,
77,
78,
79,
80,
81,
82,
83,
84,
85,
86,
87,
88,
89,
90,
91,
92,
93,
94,
95,
96,
97,
98,
99,
101,
102,
103,
104,
105,
107,
108,
109,
110,
113,
114,
115,
116,
117,
118,
119,
120,
122,
123,
124,
125,
128,
129,
130,
131,
132,
133,
134,
135,
136,
137,
138,
139,
140,
141,
142,
143,
144,
145,
146,
147,
148,
149,
150,
152,
153,
154,
155,
156,
158,
159,
160,
161,
164,
165,
166,
167,
168,
169,
170,
171,
173,
174,
175,
176,
179,
180,
181,
182,
183,
184,
185,
186,
187,
188,
189,
190,
191,
192,
193,
194,
195,
196,
197,
198,
199,
200,
201,
203,
204,
205,
206,
207,
209,
210,
211,
212,
215,
216,
217,
218,
219,
220,
221,
222,
224,
225,
226,
227,
230,
231,
232,
233,
234,
235,
236,
237,
238,
239,
240,
241,
242,
243,
244,
245,
246,
247,
248,
249,
250,
251,
252,
254,
255,
256,
257,
258,
260,
261,
262,
263,
266,
267,
268,
269,
270,
271,
272,
273,
275,
276,
277,
278,
281,
282,
283,
284,
285,
286,
287,
288,
289,
290,
291,
292,
293,
294,
295,
296,
297,
298,
299,
300,
301,
302,
303,
305,
306,
307,
308,
309,
311,
312,
313,
314,
317,
318,
319,
320,
321,
322,
323,
324,
326,
327,
328,
329,
332,
333,
334,
335,
336,
337,
338,
339,
340,
341,
342,
343,
344,
345,
346,
347,
348,
349,
350,
351,
352,
353,
354,
356,
357,
358,
359,
360,
362,
363,
364,
365,
368,
369,
370,
371,
372,
373,
374,
375,
377,
378,
379,
380,
383,
384,
385,
386,
387,
388,
389,
390,
391,
392,
393,
394,
395,
396,
397,
398,
399,
400,
401,
402,
403,
404,
405,
407,
408,
409,
410,
411,
413,
414,
415,
416,
419,
420,
421,
422,
423,
424,
425,
426,
428,
429,
430,
431,
434,
435,
436,
437,
438,
439,
440,
441,
442,
443,
444,
445,
446,
447,
448,
449,
450,
451,
452,
453,
454,
455,
456,
458,
459,
460,
461,
462,
464,
465,
466,
467,
470,
471,
472,
473,
474,
475,
476,
477,
479,
480,
481,
482,
485,
486,
487,
488,
489,
490,
491,
492,
493,
494,
495,
496,
497,
498,
499,
500,
501,
502,
503,
504,
505,
506,
507,
509,
510,
511,
512,
513,
515,
516,
517,
518,
521,
522,
523,
524,
525,
526,
527,
528,
530,
531,
532,
533,
536,
537,
538,
539,
540,
541,
542,
543,
544,
545,
546,
547,
548,
549,
550,
551,
552,
553,
554,
555,
556,
557,
558,
560,
561,
562,
563,
564,
566,
567,
568,
569,
572,
573,
574,
575,
576,
577,
578,
579,
581,
582,
583,
584,
587,
588,
589,
590,
591,
592,
593,
594,
595,
596,
597,
598,
599,
600,
601,
602,
603,
604,
605,
606,
607,
608,
609,
611,
612,
613,
614,
615,
617,
618,
619,
620,
623,
624,
625,
626,
627,
628,
629,
630,
632,
633,
634,
635,
638,
639,
640,
641,
642,
643,
644,
645,
646,
648,
649,
650,
651
],
"heads": [[562, 0, 0]],
"attrs": {"mxnet_version": ["int", 10800]}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@mxnet.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@mxnet.apache.org
For additional commands, e-mail: issues-help@mxnet.apache.org