You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2018/03/27 18:28:27 UTC
[incubator-mxnet] 02/02: Update language model (#19)
This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch nlp_toolkit
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
commit 7b604ab0d17b58f5623846ece1fa3ed846fbb82d
Author: cgwang <wa...@gmail.com>
AuthorDate: Tue Mar 27 11:24:42 2018 -0700
Update language model (#19)
* udpate lmexp
* udpate word_language_model.py
* udpate word_language_model.py
* update with lm_decay
* update word_language_model.py with new updated hiddensize, standardrnn exchange tied and dropout; update base.py with rnn_relu config; update lm.py with awd_lstm_lm_1150 pretrained setting, and with new sentiment analysis and lm example
* remove lm test file
---
example/gluon/word_language_model.py | 12 +++++++-----
python/mxnet/gluon/model_zoo/text/base.py | 2 +-
python/mxnet/gluon/model_zoo/text/lm.py | 2 +-
3 files changed, 9 insertions(+), 7 deletions(-)
diff --git a/example/gluon/word_language_model.py b/example/gluon/word_language_model.py
index 382f0b1..7dd8c69 100644
--- a/example/gluon/word_language_model.py
+++ b/example/gluon/word_language_model.py
@@ -22,7 +22,7 @@ import math
import mxnet as mx
from mxnet import gluon, autograd
from mxnet.gluon import data, text
-from mxnet.gluon.model_zoo.text.lm import SimpleRNN, AWDRNN
+from mxnet.gluon.model_zoo.text.lm import StandardRNN, AWDRNN
parser = argparse.ArgumentParser(description='MXNet Autograd RNN/LSTM Language Model on Wikitext-2.')
parser.add_argument('--model', type=str, default='lstm',
@@ -68,6 +68,8 @@ parser.add_argument('--gpus', type=str,
help='list of gpus to run, e.g. 0 or 0,2,5. empty means using cpu. (the result of multi-gpu training might be slightly different compared to single-gpu training, still need to be finalized)')
args = parser.parse_args()
+print(args)
+
###############################################################################
# Load data
@@ -82,7 +84,7 @@ train_dataset = data.text.WikiText2(segment='train', seq_len=args.bptt,
def get_frequencies(dataset):
return collections.Counter(x for tup in dataset for x in tup[0] if x)
-vocab = text.vocab.Vocabulary(get_frequencies(train_dataset))
+vocab = text.vocab.Vocabulary(get_frequencies(train_dataset), reserved_tokens=['<eos>', '<pad>'])
def index_tokens(data, label):
return vocab[data], vocab[label]
@@ -124,8 +126,8 @@ if args.weight_dropout:
model = AWDRNN(args.model, len(vocab), args.emsize, args.nhid, args.nlayers,
args.tied, args.dropout, args.weight_dropout, args.dropout_h, args.dropout_i)
else:
- model = SimpleRNN(args.model, len(vocab), args.emsize, args.nhid, args.nlayers,
- args.tied, args.dropout)
+ model = StandardRNN(args.model, len(vocab), args.emsize, args.nhid, args.nlayers, args.dropout,
+ args.tied)
model.initialize(mx.init.Xavier(), ctx=context)
@@ -169,7 +171,7 @@ def train():
for epoch in range(args.epochs):
total_L = 0.0
start_epoch_time = time.time()
- hiddens = [model.begin_state(args.batch_size, func=mx.nd.zeros, ctx=ctx) for ctx in context]
+ hiddens = [model.begin_state(args.batch_size//len(context), func=mx.nd.zeros, ctx=ctx) for ctx in context]
for i, (data, target) in enumerate(train_data):
start_batch_time = time.time()
data = data.T
diff --git a/python/mxnet/gluon/model_zoo/text/base.py b/python/mxnet/gluon/model_zoo/text/base.py
index 6f5f555..7d84bde 100644
--- a/python/mxnet/gluon/model_zoo/text/base.py
+++ b/python/mxnet/gluon/model_zoo/text/base.py
@@ -118,7 +118,7 @@ def get_rnn_cell(mode, num_layers, input_size, hidden_size,
def get_rnn_layer(mode, num_layers, input_size, hidden_size, dropout, weight_dropout):
"""create rnn layer given specs"""
if mode == 'rnn_relu':
- block = rnn.RNN(hidden_size, 'relu', num_layers, dropout=dropout,
+ block = rnn.RNN(hidden_size, num_layers, 'relu', dropout=dropout,
input_size=input_size)
elif mode == 'rnn_tanh':
block = rnn.RNN(hidden_size, num_layers, dropout=dropout,
diff --git a/python/mxnet/gluon/model_zoo/text/lm.py b/python/mxnet/gluon/model_zoo/text/lm.py
index 7607cf5..34f0609 100644
--- a/python/mxnet/gluon/model_zoo/text/lm.py
+++ b/python/mxnet/gluon/model_zoo/text/lm.py
@@ -209,7 +209,7 @@ def awd_lstm_lm_1150(dataset_name=None, vocab=None, pretrained=False, ctx=cpu(),
'tie_weights': True,
'dropout': 0.4,
'weight_drop': 0.5,
- 'drop_h': 0.3,
+ 'drop_h': 0.2,
'drop_i': 0.65}
assert all(k not in kwargs for k in predefined_args), \
"Cannot override predefined model settings."
--
To stop receiving notification emails like this one, please contact
zhasheng@apache.org.