You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2020/06/08 17:40:51 UTC
[incubator-mxnet] branch master updated: remove mx.rnn APIs (#18507)
This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 992ed3c remove mx.rnn APIs (#18507)
992ed3c is described below
commit 992ed3c1ea449fdb1f4f7010dfd05d00ae88a020
Author: Haibin Lin <li...@gmail.com>
AuthorDate: Mon Jun 8 10:39:56 2020 -0700
remove mx.rnn APIs (#18507)
* remove mx.rnn APIs
* fix test
* update test
Co-authored-by: Ubuntu <ub...@ip-172-31-37-108.ec2.internal>
Co-authored-by: Lin <ha...@a483e7be4c92.ant.amazon.com>
---
python/mxnet/__init__.py | 1 -
python/mxnet/initializer.py | 54 -
python/mxnet/rnn/__init__.py | 29 -
python/mxnet/rnn/io.py | 210 ---
python/mxnet/rnn/rnn.py | 121 --
python/mxnet/rnn/rnn_cell.py | 1441 --------------------
tests/python/gpu/test_contrib_amp.py | 35 -
tests/python/gpu/test_operator_gpu.py | 108 --
tests/python/quantization/test_quantization.py | 163 ---
tests/python/train/test_bucketing.py | 125 --
tests/python/unittest/test_contrib_control_flow.py | 193 ---
tests/python/unittest/test_module.py | 286 ----
tests/python/unittest/test_operator.py | 179 ---
tests/python/unittest/test_rnn.py | 311 -----
14 files changed, 3256 deletions(-)
diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py
index 66b23d4..ae90e4b 100644
--- a/python/mxnet/__init__.py
+++ b/python/mxnet/__init__.py
@@ -84,7 +84,6 @@ from . import image as img
from . import test_utils
-from . import rnn
from . import gluon
from . import _deferred_compute
diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py
index efec644..51cb63a 100644
--- a/python/mxnet/initializer.py
+++ b/python/mxnet/initializer.py
@@ -713,57 +713,3 @@ class LSTMBias(Initializer):
# gate of the 4 LSTM gates, we modify the according values.
num_hidden = int(arr.shape[0] / 4)
arr[num_hidden:2*num_hidden] = self.forget_bias
-
-
-@register
-class FusedRNN(Initializer):
- """Initialize parameters for fused rnn layers.
-
- Parameters
- ----------
- init : Initializer
- initializer applied to unpacked weights. Fall back to global
- initializer if None.
- num_hidden : int
- should be the same with arguments passed to FusedRNNCell.
- num_layers : int
- should be the same with arguments passed to FusedRNNCell.
- mode : str
- should be the same with arguments passed to FusedRNNCell.
- bidirectional : bool
- should be the same with arguments passed to FusedRNNCell.
- forget_bias : float
- should be the same with arguments passed to FusedRNNCell.
- """
- def __init__(self, init, num_hidden, num_layers, mode, bidirectional=False, forget_bias=1.0):
- if isinstance(init, string_types):
- klass, kwargs = json.loads(init)
- init = registry._REGISTRY[klass.lower()](**kwargs)
- super(FusedRNN, self).__init__(init=init.dumps() if init is not None else None,
- num_hidden=num_hidden, num_layers=num_layers, mode=mode,
- bidirectional=bidirectional, forget_bias=forget_bias)
- self._init = init
- self._num_hidden = num_hidden
- self._num_layers = num_layers
- self._mode = mode
- self._bidirectional = bidirectional
- self._forget_bias = forget_bias
-
- def _init_weight(self, desc, arr): # pylint: disable=arguments-differ
- from .rnn import rnn_cell
- cell = rnn_cell.FusedRNNCell(self._num_hidden, self._num_layers,
- self._mode, self._bidirectional,
- forget_bias=self._forget_bias, prefix='')
- args = cell.unpack_weights({'parameters': arr})
- for name in args:
- arg_desc = InitDesc(name, global_init=desc.global_init)
- # for lstm bias, we use a custom initializer
- # which adds a bias to the forget gate
- if self._mode == 'lstm' and name.endswith("_f_bias"):
- args[name][:] = self._forget_bias
- elif self._init is None:
- desc.global_init(arg_desc, args[name])
- else:
- self._init(arg_desc, args[name])
-
- arr[:] = cell.pack_weights(args)['parameters']
diff --git a/python/mxnet/rnn/__init__.py b/python/mxnet/rnn/__init__.py
deleted file mode 100644
index dbf382e..0000000
--- a/python/mxnet/rnn/__init__.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=wildcard-import
-"""Recurrent neural network module."""
-
-from . import rnn_cell
-from .rnn_cell import *
-
-from . import rnn
-from .rnn import *
-
-from . import io
-from .io import *
diff --git a/python/mxnet/rnn/io.py b/python/mxnet/rnn/io.py
deleted file mode 100644
index a10714b8..0000000
--- a/python/mxnet/rnn/io.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=too-many-arguments, too-many-locals
-"""Definition of various recurrent neural network cells."""
-
-import bisect
-import random
-import numpy as np
-
-from ..io import DataIter, DataBatch, DataDesc
-from .. import ndarray
-
-def encode_sentences(sentences, vocab=None, invalid_label=-1, invalid_key='\n',
- start_label=0, unknown_token=None):
- """Encode sentences and (optionally) build a mapping
- from string tokens to integer indices. Unknown keys
- will be added to vocabulary.
-
- Parameters
- ----------
- sentences : list of list of str
- A list of sentences to encode. Each sentence
- should be a list of string tokens.
- vocab : None or dict of str -> int
- Optional input Vocabulary
- invalid_label : int, default -1
- Index for invalid token, like <end-of-sentence>
- invalid_key : str, default '\\n'
- Key for invalid token. Use '\\n' for end
- of sentence by default.
- start_label : int
- lowest index.
- unknown_token: str
- Symbol to represent unknown token.
- If not specified, unknown token will be skipped.
-
- Returns
- -------
- result : list of list of int
- encoded sentences
- vocab : dict of str -> int
- result vocabulary
- """
- idx = start_label
- if vocab is None:
- vocab = {invalid_key: invalid_label}
- new_vocab = True
- else:
- new_vocab = False
- res = []
- for sent in sentences:
- coded = []
- for word in sent:
- if word not in vocab:
- assert (new_vocab or unknown_token), "Unknown token %s"%word
- if idx == invalid_label:
- idx += 1
- if unknown_token:
- word = unknown_token
- vocab[word] = idx
- idx += 1
- coded.append(vocab[word])
- res.append(coded)
-
- return res, vocab
-
-class BucketSentenceIter(DataIter):
- """Simple bucketing iterator for language model.
- The label at each sequence step is the following token
- in the sequence.
-
- Parameters
- ----------
- sentences : list of list of int
- Encoded sentences.
- batch_size : int
- Batch size of the data.
- invalid_label : int, optional
- Key for invalid label, e.g. <end-of-sentence>. The default is -1.
- dtype : str, optional
- Data type of the encoding. The default data type is 'float32'.
- buckets : list of int, optional
- Size of the data buckets. Automatically generated if None.
- data_name : str, optional
- Name of the data. The default name is 'data'.
- label_name : str, optional
- Name of the label. The default name is 'softmax_label'.
- layout : str, optional
- Format of data and label. 'NT' means (batch_size, length)
- and 'TN' means (length, batch_size).
- """
- def __init__(self, sentences, batch_size, buckets=None, invalid_label=-1,
- data_name='data', label_name='softmax_label', dtype='float32',
- layout='NT'):
- super(BucketSentenceIter, self).__init__()
- if not buckets:
- buckets = [i for i, j in enumerate(np.bincount([len(s) for s in sentences]))
- if j >= batch_size]
- buckets.sort()
-
- ndiscard = 0
- self.data = [[] for _ in buckets]
- valid_buckets = {}
- for item in range(len(buckets)):
- valid_buckets[item] = 0
-
- for i, sent in enumerate(sentences):
- buck = bisect.bisect_left(buckets, len(sent))
- valid_buckets[buck] = 1
- if buck == len(buckets):
- ndiscard += 1
- continue
- buff = np.full((buckets[buck],), invalid_label, dtype=dtype)
- buff[:len(sent)] = sent
- self.data[buck].append(buff)
- buckets = [j for i, j in enumerate(buckets) if valid_buckets[i] == 1]
- self.data = [np.asarray(i, dtype=dtype) for i in self.data if i]
-
- print("WARNING: discarded %d sentences longer than the largest bucket."%ndiscard)
-
- self.batch_size = batch_size
- self.buckets = buckets
- self.data_name = data_name
- self.label_name = label_name
- self.dtype = dtype
- self.invalid_label = invalid_label
- self.nddata = []
- self.ndlabel = []
- self.major_axis = layout.find('N')
- self.layout = layout
- self.default_bucket_key = max(buckets)
-
- if self.major_axis == 0:
- self.provide_data = [DataDesc(
- name=self.data_name, shape=(batch_size, self.default_bucket_key),
- layout=self.layout)]
- self.provide_label = [DataDesc(
- name=self.label_name, shape=(batch_size, self.default_bucket_key),
- layout=self.layout)]
- elif self.major_axis == 1:
- self.provide_data = [DataDesc(
- name=self.data_name, shape=(self.default_bucket_key, batch_size),
- layout=self.layout)]
- self.provide_label = [DataDesc(
- name=self.label_name, shape=(self.default_bucket_key, batch_size),
- layout=self.layout)]
- else:
- raise ValueError("Invalid layout %s: Must by NT (batch major) or TN (time major)")
-
- self.idx = []
- for i, buck in enumerate(self.data):
- self.idx.extend([(i, j) for j in range(0, len(buck) - batch_size + 1, batch_size)])
- self.curr_idx = 0
-
- self.reset()
-
- def reset(self):
- """Resets the iterator to the beginning of the data."""
- self.curr_idx = 0
- random.shuffle(self.idx)
- for buck in self.data:
- np.random.shuffle(buck)
-
- self.nddata = []
- self.ndlabel = []
- for buck in self.data:
- label = np.empty_like(buck)
- label[:, :-1] = buck[:, 1:]
- label[:, -1] = self.invalid_label
- self.nddata.append(ndarray.array(buck, dtype=self.dtype))
- self.ndlabel.append(ndarray.array(label, dtype=self.dtype))
-
- def next(self):
- """Returns the next batch of data."""
- if self.curr_idx == len(self.idx):
- raise StopIteration
- i, j = self.idx[self.curr_idx]
- self.curr_idx += 1
-
- if self.major_axis == 1:
- data = self.nddata[i][j:j+self.batch_size].T
- label = self.ndlabel[i][j:j+self.batch_size].T
- else:
- data = self.nddata[i][j:j+self.batch_size]
- label = self.ndlabel[i][j:j+self.batch_size]
-
- return DataBatch([data], [label], pad=0,
- bucket_key=self.buckets[i],
- provide_data=[DataDesc(
- name=self.data_name, shape=data.shape,
- layout=self.layout)],
- provide_label=[DataDesc(
- name=self.label_name, shape=label.shape,
- layout=self.layout)])
diff --git a/python/mxnet/rnn/rnn.py b/python/mxnet/rnn/rnn.py
deleted file mode 100644
index 0255c55..0000000
--- a/python/mxnet/rnn/rnn.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=too-many-arguments, no-member
-"""Functions for constructing recurrent neural networks."""
-import warnings
-
-from ..model import save_checkpoint, load_checkpoint
-from .rnn_cell import BaseRNNCell
-
-def rnn_unroll(cell, length, inputs=None, begin_state=None, input_prefix='', layout='NTC'):
- """Deprecated. Please use cell.unroll instead"""
- warnings.warn('rnn_unroll is deprecated. Please call cell.unroll directly.')
- return cell.unroll(length=length, inputs=inputs, begin_state=begin_state,
- input_prefix=input_prefix, layout=layout)
-
-def save_rnn_checkpoint(cells, prefix, epoch, symbol, arg_params, aux_params):
- """Save checkpoint for model using RNN cells.
- Unpacks weight before saving.
-
- Parameters
- ----------
- cells : mxnet.rnn.RNNCell or list of RNNCells
- The RNN cells used by this symbol.
- prefix : str
- Prefix of model name.
- epoch : int
- The epoch number of the model.
- symbol : Symbol
- The input symbol
- arg_params : dict of str to NDArray
- Model parameter, dict of name to NDArray of net's weights.
- aux_params : dict of str to NDArray
- Model parameter, dict of name to NDArray of net's auxiliary states.
-
- Notes
- -----
- - ``prefix-symbol.json`` will be saved for symbol.
- - ``prefix-epoch.params`` will be saved for parameters.
- """
- if isinstance(cells, BaseRNNCell):
- cells = [cells]
- for cell in cells:
- arg_params = cell.unpack_weights(arg_params)
- save_checkpoint(prefix, epoch, symbol, arg_params, aux_params)
-
-def load_rnn_checkpoint(cells, prefix, epoch):
- """Load model checkpoint from file.
- Pack weights after loading.
-
- Parameters
- ----------
- cells : mxnet.rnn.RNNCell or list of RNNCells
- The RNN cells used by this symbol.
- prefix : str
- Prefix of model name.
- epoch : int
- Epoch number of model we would like to load.
-
- Returns
- -------
- symbol : Symbol
- The symbol configuration of computation network.
- arg_params : dict of str to NDArray
- Model parameter, dict of name to NDArray of net's weights.
- aux_params : dict of str to NDArray
- Model parameter, dict of name to NDArray of net's auxiliary states.
-
- Notes
- -----
- - symbol will be loaded from ``prefix-symbol.json``.
- - parameters will be loaded from ``prefix-epoch.params``.
- """
- sym, arg, aux = load_checkpoint(prefix, epoch)
- if isinstance(cells, BaseRNNCell):
- cells = [cells]
- for cell in cells:
- arg = cell.pack_weights(arg)
-
- return sym, arg, aux
-
-def do_rnn_checkpoint(cells, prefix, period=1):
- """Make a callback to checkpoint Module to prefix every epoch.
- unpacks weights used by cells before saving.
-
- Parameters
- ----------
- cells : mxnet.rnn.RNNCell or list of RNNCells
- The RNN cells used by this symbol.
- prefix : str
- The file prefix to checkpoint to
- period : int
- How many epochs to wait before checkpointing. Default is 1.
-
- Returns
- -------
- callback : function
- The callback function that can be passed as iter_end_callback to fit.
- """
- period = int(max(1, period))
- # pylint: disable=unused-argument
- def _callback(iter_no, sym=None, arg=None, aux=None):
- """The checkpoint function."""
- if (iter_no + 1) % period == 0:
- save_rnn_checkpoint(cells, prefix, iter_no+1, sym, arg, aux)
- return _callback
diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py
deleted file mode 100644
index f6f99e8..0000000
--- a/python/mxnet/rnn/rnn_cell.py
+++ /dev/null
@@ -1,1441 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=no-member, invalid-name, protected-access, no-self-use
-# pylint: disable=too-many-branches, too-many-arguments, no-self-use
-# pylint: disable=too-many-lines
-"""Definition of various recurrent neural network cells."""
-
-import warnings
-import functools
-
-from .. import symbol, init, ndarray
-from ..base import string_types, numeric_types
-
-
-def _cells_state_shape(cells):
- return sum([c.state_shape for c in cells], [])
-
-def _cells_state_info(cells):
- return sum([c.state_info for c in cells], [])
-
-def _cells_begin_state(cells, **kwargs):
- return sum([c.begin_state(**kwargs) for c in cells], [])
-
-def _cells_unpack_weights(cells, args):
- for cell in cells:
- args = cell.unpack_weights(args)
- return args
-
-def _cells_pack_weights(cells, args):
- for cell in cells:
- args = cell.pack_weights(args)
- return args
-
-def _normalize_sequence(length, inputs, layout, merge, in_layout=None):
- assert inputs is not None, \
- "unroll(inputs=None) has been deprecated. " \
- "Please create input variables outside unroll."
-
- axis = layout.find('T')
- in_axis = in_layout.find('T') if in_layout is not None else axis
- if isinstance(inputs, symbol.Symbol):
- if merge is False:
- assert len(inputs.list_outputs()) == 1, \
- "unroll doesn't allow grouped symbol as input. Please convert " \
- "to list with list(inputs) first or let unroll handle splitting."
- inputs = list(symbol.split(inputs, axis=in_axis, num_outputs=length,
- squeeze_axis=1))
- else:
- assert length is None or len(inputs) == length
- if merge is True:
- inputs = [symbol.expand_dims(i, axis=axis) for i in inputs]
- inputs = symbol.Concat(*inputs, dim=axis)
- in_axis = axis
-
- if isinstance(inputs, symbol.Symbol) and axis != in_axis:
- inputs = symbol.swapaxes(inputs, dim0=axis, dim1=in_axis)
-
- return inputs, axis
-
-
-class RNNParams(object):
- """Container for holding variables.
- Used by RNN cells for parameter sharing between cells.
-
- Parameters
- ----------
- prefix : str
- Names of all variables created by this container will
- be prepended with prefix.
- """
- def __init__(self, prefix=''):
- self._prefix = prefix
- self._params = {}
-
- def get(self, name, **kwargs):
- """Get the variable given a name if one exists or create a new one if missing.
-
- Parameters
- ----------
- name : str
- name of the variable
- **kwargs :
- more arguments that's passed to symbol.Variable
- """
- name = self._prefix + name
- if name not in self._params:
- self._params[name] = symbol.Variable(name, **kwargs)
- return self._params[name]
-
-
-class BaseRNNCell(object):
- """Abstract base class for RNN cells
-
- Parameters
- ----------
- prefix : str, optional
- Prefix for names of layers
- (this prefix is also used for names of weights if `params` is None
- i.e. if `params` are being created and not reused)
- params : RNNParams, default None.
- Container for weight sharing between cells.
- A new RNNParams container is created if `params` is None.
- """
- def __init__(self, prefix='', params=None):
- if params is None:
- params = RNNParams(prefix)
- self._own_params = True
- else:
- self._own_params = False
- self._prefix = prefix
- self._params = params
- self._modified = False
-
- self.reset()
-
- def reset(self):
- """Reset before re-using the cell for another graph."""
- self._init_counter = -1
- self._counter = -1
- if hasattr(self, '_cells'):
- for cell in self._cells:
- cell.reset()
-
- def __call__(self, inputs, states):
- """Unroll the RNN for one time step.
-
- Parameters
- ----------
- inputs : sym.Variable
- input symbol, 2D, batch * num_units
- states : list of sym.Variable
- RNN state from previous step or the output of begin_state().
-
- Returns
- -------
- output : Symbol
- Symbol corresponding to the output from the RNN when unrolling
- for a single time step.
- states : nested list of Symbol
- The new state of this RNN after this unrolling.
- The type of this symbol is same as the output of begin_state().
- This can be used as input state to the next time step
- of this RNN.
-
- See Also
- --------
- begin_state: This function can provide the states for the first time step.
- unroll: This function unrolls an RNN for a given number of (>=1) time steps.
- """
- raise NotImplementedError()
-
- @property
- def params(self):
- """Parameters of this cell"""
- self._own_params = False
- return self._params
-
- @property
- def state_info(self):
- """shape and layout information of states"""
- raise NotImplementedError()
-
- @property
- def state_shape(self):
- """shape(s) of states"""
- return [ele['shape'] for ele in self.state_info]
-
- @property
- def _gate_names(self):
- """name(s) of gates"""
- return ()
-
- def begin_state(self, func=symbol.zeros, **kwargs):
- """Initial state for this cell.
-
- Parameters
- ----------
- func : callable, default symbol.zeros
- Function for creating initial state. Can be symbol.zeros,
- symbol.uniform, symbol.Variable etc.
- Use symbol.Variable if you want to directly
- feed input as states.
- **kwargs :
- more keyword arguments passed to func. For example
- mean, std, dtype, etc.
-
- Returns
- -------
- states : nested list of Symbol
- Starting states for the first RNN step.
- """
- assert not self._modified, \
- "After applying modifier cells (e.g. DropoutCell) the base " \
- "cell cannot be called directly. Call the modifier cell instead."
- states = []
- for info in self.state_info:
- self._init_counter += 1
- if info is None:
- state = func(name='%sbegin_state_%d'%(self._prefix, self._init_counter),
- **kwargs)
- else:
- kwargs.update(info)
- state = func(name='%sbegin_state_%d'%(self._prefix, self._init_counter),
- **kwargs)
- states.append(state)
- return states
-
- def unpack_weights(self, args):
- """Unpack fused weight matrices into separate
- weight matrices.
-
- For example, say you use a module object `mod` to run a network that has an lstm cell.
- In `mod.get_params()[0]`, the lstm parameters are all represented as a single big vector.
- `cell.unpack_weights(mod.get_params()[0])` will unpack this vector into a dictionary of
- more readable lstm parameters - c, f, i, o gates for i2h (input to hidden) and
- h2h (hidden to hidden) weights.
-
- Parameters
- ----------
- args : dict of str -> NDArray
- Dictionary containing packed weights.
- usually from `Module.get_params()[0]`.
-
- Returns
- -------
- args : dict of str -> NDArray
- Dictionary with unpacked weights associated with
- this cell.
-
- See Also
- --------
- pack_weights: Performs the reverse operation of this function.
- """
- args = args.copy()
- if not self._gate_names:
- return args
- h = self._num_hidden
- for group_name in ['i2h', 'h2h']:
- weight = args.pop('%s%s_weight'%(self._prefix, group_name))
- bias = args.pop('%s%s_bias' % (self._prefix, group_name))
- for j, gate in enumerate(self._gate_names):
- wname = '%s%s%s_weight' % (self._prefix, group_name, gate)
- args[wname] = weight[j*h:(j+1)*h].copy()
- bname = '%s%s%s_bias' % (self._prefix, group_name, gate)
- args[bname] = bias[j*h:(j+1)*h].copy()
- return args
-
- def pack_weights(self, args):
- """Pack separate weight matrices into a single packed
- weight.
-
- Parameters
- ----------
- args : dict of str -> NDArray
- Dictionary containing unpacked weights.
-
- Returns
- -------
- args : dict of str -> NDArray
- Dictionary with packed weights associated with
- this cell.
- """
- args = args.copy()
- if not self._gate_names:
- return args
- for group_name in ['i2h', 'h2h']:
- weight = []
- bias = []
- for gate in self._gate_names:
- wname = '%s%s%s_weight'%(self._prefix, group_name, gate)
- weight.append(args.pop(wname))
- bname = '%s%s%s_bias'%(self._prefix, group_name, gate)
- bias.append(args.pop(bname))
- args['%s%s_weight'%(self._prefix, group_name)] = ndarray.concatenate(weight)
- args['%s%s_bias'%(self._prefix, group_name)] = ndarray.concatenate(bias)
- return args
-
- def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
- """Unroll an RNN cell across time steps.
-
- Parameters
- ----------
- length : int
- Number of steps to unroll.
- inputs : Symbol, list of Symbol, or None
- If `inputs` is a single Symbol (usually the output
- of Embedding symbol), it should have shape
- (batch_size, length, ...) if layout == 'NTC',
- or (length, batch_size, ...) if layout == 'TNC'.
-
- If `inputs` is a list of symbols (usually output of
- previous unroll), they should all have shape
- (batch_size, ...).
- begin_state : nested list of Symbol, default None
- Input states created by `begin_state()`
- or output state of another cell.
- Created from `begin_state()` if None.
- layout : str, optional
- `layout` of input symbol. Only used if inputs
- is a single Symbol.
- merge_outputs : bool, optional
- If False, return outputs as a list of Symbols.
- If True, concatenate output across time steps
- and return a single symbol with shape
- (batch_size, length, ...) if layout == 'NTC',
- or (length, batch_size, ...) if layout == 'TNC'.
- If None, output whatever is faster.
-
- Returns
- -------
- outputs : list of Symbol or Symbol
- Symbol (if `merge_outputs` is True) or list of Symbols
- (if `merge_outputs` is False) corresponding to the output from
- the RNN from this unrolling.
-
- states : nested list of Symbol
- The new state of this RNN after this unrolling.
- The type of this symbol is same as the output of begin_state().
- """
- self.reset()
-
- inputs, _ = _normalize_sequence(length, inputs, layout, False)
- if begin_state is None:
- begin_state = self.begin_state()
-
- states = begin_state
- outputs = []
- for i in range(length):
- output, states = self(inputs[i], states)
- outputs.append(output)
-
- outputs, _ = _normalize_sequence(length, outputs, layout, merge_outputs)
-
- return outputs, states
-
- #pylint: disable=no-self-use
- def _get_activation(self, inputs, activation, **kwargs):
- """Get activation function. Convert if is string"""
- if isinstance(activation, string_types):
- return symbol.Activation(inputs, act_type=activation, **kwargs)
- else:
- return activation(inputs, **kwargs)
-
-
-class RNNCell(BaseRNNCell):
- """Simple recurrent neural network cell.
-
- Parameters
- ----------
- num_hidden : int
- Number of units in output symbol.
- activation : str or Symbol, default 'tanh'
- Type of activation function. Options are 'relu' and 'tanh'.
- prefix : str, default ``'rnn_'``
- Prefix for name of layers (and name of weight if params is None).
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- """
- def __init__(self, num_hidden, activation='tanh', prefix='rnn_', params=None):
- super(RNNCell, self).__init__(prefix=prefix, params=params)
- self._num_hidden = num_hidden
- self._activation = activation
- self._iW = self.params.get('i2h_weight')
- self._iB = self.params.get('i2h_bias')
- self._hW = self.params.get('h2h_weight')
- self._hB = self.params.get('h2h_bias')
-
- @property
- def state_info(self):
- return [{'shape': (0, self._num_hidden), '__layout__': 'NC'}]
-
- @property
- def _gate_names(self):
- return ('',)
-
- def __call__(self, inputs, states):
- self._counter += 1
- name = '%st%d_'%(self._prefix, self._counter)
- i2h = symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB,
- num_hidden=self._num_hidden,
- name='%si2h'%name)
- h2h = symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB,
- num_hidden=self._num_hidden,
- name='%sh2h'%name)
- output = self._get_activation(i2h + h2h, self._activation,
- name='%sout'%name)
-
- return output, [output]
-
-
-class LSTMCell(BaseRNNCell):
- """Long-Short Term Memory (LSTM) network cell.
-
- Parameters
- ----------
- num_hidden : int
- Number of units in output symbol.
- prefix : str, default ``'lstm_'``
- Prefix for name of layers (and name of weight if params is None).
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- forget_bias : bias added to forget gate, default 1.0.
- Jozefowicz et al. 2015 recommends setting this to 1.0
- """
- def __init__(self, num_hidden, prefix='lstm_', params=None, forget_bias=1.0):
- super(LSTMCell, self).__init__(prefix=prefix, params=params)
-
- self._num_hidden = num_hidden
- self._iW = self.params.get('i2h_weight')
- self._hW = self.params.get('h2h_weight')
- # we add the forget_bias to i2h_bias, this adds the bias to the forget gate activation
- self._iB = self.params.get('i2h_bias', init=init.LSTMBias(forget_bias=forget_bias))
- self._hB = self.params.get('h2h_bias')
-
- @property
- def state_info(self):
- return [{'shape': (0, self._num_hidden), '__layout__': 'NC'},
- {'shape': (0, self._num_hidden), '__layout__': 'NC'}]
-
- @property
- def _gate_names(self):
- return ['_i', '_f', '_c', '_o']
-
- def __call__(self, inputs, states):
- self._counter += 1
- name = '%st%d_'%(self._prefix, self._counter)
- i2h = symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB,
- num_hidden=self._num_hidden*4,
- name='%si2h'%name)
- h2h = symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB,
- num_hidden=self._num_hidden*4,
- name='%sh2h'%name)
- gates = i2h + h2h
- slice_gates = symbol.SliceChannel(gates, num_outputs=4,
- name="%sslice"%name)
- in_gate = symbol.Activation(slice_gates[0], act_type="sigmoid",
- name='%si'%name)
- forget_gate = symbol.Activation(slice_gates[1], act_type="sigmoid",
- name='%sf'%name)
- in_transform = symbol.Activation(slice_gates[2], act_type="tanh",
- name='%sc'%name)
- out_gate = symbol.Activation(slice_gates[3], act_type="sigmoid",
- name='%so'%name)
- next_c = symbol._internal._plus(forget_gate * states[1], in_gate * in_transform,
- name='%sstate'%name)
- next_c._set_attr(force_mirroring='0')
- # Cell states are excluded from being mirrored. The reason is because
- # they do not pass through the fully-connected layers and will
- # significantly increase the overall mirroring depth, incurring large
- # performance overhead.
- next_h = symbol._internal._mul(out_gate, symbol.Activation(next_c, act_type="tanh"),
- name='%sout'%name)
-
- return next_h, [next_h, next_c]
-
-
-class GRUCell(BaseRNNCell):
- """Gated Rectified Unit (GRU) network cell.
- Note: this is an implementation of the cuDNN version of GRUs
- (slight modification compared to Cho et al. 2014).
-
- Parameters
- ----------
- num_hidden : int
- Number of units in output symbol.
- prefix : str, default ``'gru_'``
- Prefix for name of layers (and name of weight if params is None).
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- """
- def __init__(self, num_hidden, prefix='gru_', params=None):
- super(GRUCell, self).__init__(prefix=prefix, params=params)
- self._num_hidden = num_hidden
- self._iW = self.params.get("i2h_weight")
- self._iB = self.params.get("i2h_bias")
- self._hW = self.params.get("h2h_weight")
- self._hB = self.params.get("h2h_bias")
-
- @property
- def state_info(self):
- return [{'shape': (0, self._num_hidden),
- '__layout__': 'NC'}]
-
- @property
- def _gate_names(self):
- return ['_r', '_z', '_o']
-
- def __call__(self, inputs, states):
- # pylint: disable=too-many-locals
- self._counter += 1
-
- seq_idx = self._counter
- name = '%st%d_' % (self._prefix, seq_idx)
- prev_state_h = states[0]
-
- i2h = symbol.FullyConnected(data=inputs,
- weight=self._iW,
- bias=self._iB,
- num_hidden=self._num_hidden * 3,
- name="%s_i2h" % name)
- h2h = symbol.FullyConnected(data=prev_state_h,
- weight=self._hW,
- bias=self._hB,
- num_hidden=self._num_hidden * 3,
- name="%s_h2h" % name)
- # pylint: disable=unbalanced-tuple-unpacking
- i2h_r, i2h_z, i2h = symbol.SliceChannel(i2h, num_outputs=3, name="%s_i2h_slice" % name)
- h2h_r, h2h_z, h2h = symbol.SliceChannel(h2h, num_outputs=3, name="%s_h2h_slice" % name)
-
- reset_gate = symbol.Activation(i2h_r + h2h_r, act_type="sigmoid",
- name="%s_r_act" % name)
- update_gate = symbol.Activation(i2h_z + h2h_z, act_type="sigmoid",
- name="%s_z_act" % name)
-
- next_h_tmp = symbol.Activation(i2h + reset_gate * h2h, act_type="tanh",
- name="%s_h_act" % name)
-
- next_h = symbol._internal._plus((1. - update_gate) * next_h_tmp, update_gate * prev_state_h,
- name='%sout' % name)
-
- return next_h, [next_h]
-
-
-class FusedRNNCell(BaseRNNCell):
- """Fusing RNN layers across time step into one kernel.
- Improves speed but is less flexible. Currently only
- supported if using cuDNN on GPU.
-
- Parameters
- ----------
- num_hidden : int
- Number of units in output symbol.
- num_layers : int, default 1
- Number of layers in the cell.
- mode : str, default 'lstm'
- Type of RNN. options are 'rnn_relu', 'rnn_tanh', 'lstm', 'gru'.
- bidirectional : bool, default False
- Whether to use bidirectional unroll. The output dimension size is doubled if bidrectional.
- dropout : float, default 0.
- Fraction of the input that gets dropped out during training time.
- get_next_state : bool, default False
- Whether to return the states that can be used as starting states next time.
- forget_bias : bias added to forget gate, default 1.0.
- Jozefowicz et al. 2015 recommends setting this to 1.0
- prefix : str, default ``'$mode_'`` such as ``'lstm_'``
- Prefix for names of layers
- (this prefix is also used for names of weights if `params` is None
- i.e. if `params` are being created and not reused)
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- """
- def __init__(self, num_hidden, num_layers=1, mode='lstm', bidirectional=False,
- dropout=0., get_next_state=False, forget_bias=1.0,
- prefix=None, params=None):
- if prefix is None:
- prefix = '%s_'%mode
- super(FusedRNNCell, self).__init__(prefix=prefix, params=params)
- self._num_hidden = num_hidden
- self._num_layers = num_layers
- self._mode = mode
- self._bidirectional = bidirectional
- self._dropout = dropout
- self._get_next_state = get_next_state
- self._directions = ['l', 'r'] if bidirectional else ['l']
-
- initializer = init.FusedRNN(None, num_hidden, num_layers, mode,
- bidirectional, forget_bias)
- self._parameter = self.params.get('parameters', init=initializer)
-
- @property
- def state_info(self):
- b = self._bidirectional + 1
- n = (self._mode == 'lstm') + 1
- return [{'shape': (b*self._num_layers, 0, self._num_hidden), '__layout__': 'LNC'}
- for _ in range(n)]
-
- @property
- def _gate_names(self):
- return {'rnn_relu': [''],
- 'rnn_tanh': [''],
- 'lstm': ['_i', '_f', '_c', '_o'],
- 'gru': ['_r', '_z', '_o']}[self._mode]
-
- @property
- def _num_gates(self):
- return len(self._gate_names)
-
- def _slice_weights(self, arr, li, lh):
- """slice fused rnn weights"""
- args = {}
- gate_names = self._gate_names
- directions = self._directions
-
- b = len(directions)
- p = 0
- for layer in range(self._num_layers):
- for direction in directions:
- for gate in gate_names:
- name = '%s%s%d_i2h%s_weight'%(self._prefix, direction, layer, gate)
- if layer > 0:
- size = b*lh*lh
- args[name] = arr[p:p+size].reshape((lh, b*lh))
- else:
- size = li*lh
- args[name] = arr[p:p+size].reshape((lh, li))
- p += size
- for gate in gate_names:
- name = '%s%s%d_h2h%s_weight'%(self._prefix, direction, layer, gate)
- size = lh**2
- args[name] = arr[p:p+size].reshape((lh, lh))
- p += size
-
- for layer in range(self._num_layers):
- for direction in directions:
- for gate in gate_names:
- name = '%s%s%d_i2h%s_bias'%(self._prefix, direction, layer, gate)
- args[name] = arr[p:p+lh]
- p += lh
- for gate in gate_names:
- name = '%s%s%d_h2h%s_bias'%(self._prefix, direction, layer, gate)
- args[name] = arr[p:p+lh]
- p += lh
-
- assert p == arr.size, "Invalid parameters size for FusedRNNCell"
- return args
-
- def unpack_weights(self, args):
- args = args.copy()
- arr = args.pop(self._parameter.name)
- b = len(self._directions)
- m = self._num_gates
- h = self._num_hidden
- num_input = arr.size//b//h//m - (self._num_layers - 1)*(h+b*h+2) - h - 2
-
- nargs = self._slice_weights(arr, num_input, self._num_hidden)
- args.update({name: nd.copy() for name, nd in nargs.items()})
- return args
-
- def pack_weights(self, args):
- args = args.copy()
- b = self._bidirectional + 1
- m = self._num_gates
- c = self._gate_names
- h = self._num_hidden
- w0 = args['%sl0_i2h%s_weight'%(self._prefix, c[0])]
- num_input = w0.shape[1]
- total = (num_input+h+2)*h*m*b + (self._num_layers-1)*m*h*(h+b*h+2)*b
-
- arr = ndarray.zeros((total,), ctx=w0.context, dtype=w0.dtype)
- for name, nd in self._slice_weights(arr, num_input, h).items():
- nd[:] = args.pop(name)
- args[self._parameter.name] = arr
- return args
-
- def __call__(self, inputs, states):
- raise NotImplementedError("FusedRNNCell cannot be stepped. Please use unroll")
-
- def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
- self.reset()
-
- inputs, axis = _normalize_sequence(length, inputs, layout, True)
- if axis == 1:
- warnings.warn("NTC layout detected. Consider using "
- "TNC for FusedRNNCell for faster speed")
- inputs = symbol.swapaxes(inputs, dim1=0, dim2=1)
- else:
- assert axis == 0, "Unsupported layout %s"%layout
- if begin_state is None:
- begin_state = self.begin_state()
-
- states = begin_state
- if self._mode == 'lstm':
- states = {'state': states[0], 'state_cell': states[1]} # pylint: disable=redefined-variable-type
- else:
- states = {'state': states[0]}
-
- rnn = symbol.RNN(data=inputs, parameters=self._parameter,
- state_size=self._num_hidden, num_layers=self._num_layers,
- bidirectional=self._bidirectional, p=self._dropout,
- state_outputs=self._get_next_state,
- mode=self._mode, name=self._prefix+'rnn',
- **states)
-
- attr = {'__layout__' : 'LNC'}
- if not self._get_next_state:
- outputs, states = rnn, []
- elif self._mode == 'lstm':
- rnn[1]._set_attr(**attr)
- rnn[2]._set_attr(**attr)
- outputs, states = rnn[0], [rnn[1], rnn[2]]
- else:
- rnn[1]._set_attr(**attr)
- outputs, states = rnn[0], [rnn[1]]
-
- if axis == 1:
- outputs = symbol.swapaxes(outputs, dim1=0, dim2=1)
-
- outputs, _ = _normalize_sequence(length, outputs, layout, merge_outputs)
-
- return outputs, states
-
- def unfuse(self):
- """Unfuse the fused RNN in to a stack of rnn cells.
-
- Returns
- -------
- cell : mxnet.rnn.SequentialRNNCell
- unfused cell that can be used for stepping, and can run on CPU.
- """
- stack = SequentialRNNCell()
- get_cell = {'rnn_relu': lambda cell_prefix: RNNCell(self._num_hidden,
- activation='relu',
- prefix=cell_prefix),
- 'rnn_tanh': lambda cell_prefix: RNNCell(self._num_hidden,
- activation='tanh',
- prefix=cell_prefix),
- 'lstm': lambda cell_prefix: LSTMCell(self._num_hidden,
- prefix=cell_prefix),
- 'gru': lambda cell_prefix: GRUCell(self._num_hidden,
- prefix=cell_prefix)}[self._mode]
- for i in range(self._num_layers):
- if self._bidirectional:
- stack.add(BidirectionalCell(
- get_cell('%sl%d_'%(self._prefix, i)),
- get_cell('%sr%d_'%(self._prefix, i)),
- output_prefix='%sbi_l%d_'%(self._prefix, i)))
- else:
- stack.add(get_cell('%sl%d_'%(self._prefix, i)))
-
- if self._dropout > 0 and i != self._num_layers - 1:
- stack.add(DropoutCell(self._dropout, prefix='%s_dropout%d_'%(self._prefix, i)))
-
- return stack
-
-
-class SequentialRNNCell(BaseRNNCell):
- """Sequantially stacking multiple RNN cells.
-
- Parameters
- ----------
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- """
- def __init__(self, params=None):
- super(SequentialRNNCell, self).__init__(prefix='', params=params)
- self._override_cell_params = params is not None
- self._cells = []
-
- def add(self, cell):
- """Append a cell into the stack.
-
- Parameters
- ----------
- cell : BaseRNNCell
- The cell to be appended. During unroll, previous cell's output (or raw inputs if
- no previous cell) is used as the input to this cell.
- """
- self._cells.append(cell)
- if self._override_cell_params:
- assert cell._own_params, \
- "Either specify params for SequentialRNNCell " \
- "or child cells, not both."
- cell.params._params.update(self.params._params)
- self.params._params.update(cell.params._params)
-
- @property
- def state_info(self):
- return _cells_state_info(self._cells)
-
- def begin_state(self, **kwargs): # pylint: disable=arguments-differ
- assert not self._modified, \
- "After applying modifier cells (e.g. ZoneoutCell) the base " \
- "cell cannot be called directly. Call the modifier cell instead."
- return _cells_begin_state(self._cells, **kwargs)
-
- def unpack_weights(self, args):
- return _cells_unpack_weights(self._cells, args)
-
- def pack_weights(self, args):
- return _cells_pack_weights(self._cells, args)
-
- def __call__(self, inputs, states):
- self._counter += 1
- next_states = []
- p = 0
- for cell in self._cells:
- assert not isinstance(cell, BidirectionalCell)
- n = len(cell.state_info)
- state = states[p:p+n]
- p += n
- inputs, state = cell(inputs, state)
- next_states.append(state)
- return inputs, sum(next_states, [])
-
- def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
- self.reset()
-
- num_cells = len(self._cells)
- if begin_state is None:
- begin_state = self.begin_state()
-
- p = 0
- next_states = []
- for i, cell in enumerate(self._cells):
- n = len(cell.state_info)
- states = begin_state[p:p+n]
- p += n
- inputs, states = cell.unroll(length, inputs=inputs, begin_state=states, layout=layout,
- merge_outputs=None if i < num_cells-1 else merge_outputs)
- next_states.extend(states)
-
- return inputs, next_states
-
-
-class DropoutCell(BaseRNNCell):
- """Apply dropout on input.
-
- Parameters
- ----------
- dropout : float
- Percentage of elements to drop out, which
- is 1 - percentage to retain.
- prefix : str, default ``'dropout_'``
- Prefix for names of layers
- (this prefix is also used for names of weights if `params` is None
- i.e. if `params` are being created and not reused)
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- """
- def __init__(self, dropout, prefix='dropout_', params=None):
- super(DropoutCell, self).__init__(prefix, params)
- assert isinstance(dropout, numeric_types), "dropout probability must be a number"
- self.dropout = dropout
-
- @property
- def state_info(self):
- return []
-
- def __call__(self, inputs, states):
- if self.dropout > 0:
- inputs = symbol.Dropout(data=inputs, p=self.dropout)
- return inputs, states
-
- def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
- self.reset()
- inputs, _ = _normalize_sequence(length, inputs, layout, merge_outputs)
- if isinstance(inputs, symbol.Symbol):
- return self(inputs, [])
- else:
- return super(DropoutCell, self).unroll(
- length, inputs, begin_state=begin_state, layout=layout,
- merge_outputs=merge_outputs)
-
-
-class ModifierCell(BaseRNNCell):
- """Base class for modifier cells. A modifier
- cell takes a base cell, apply modifications
- on it (e.g. Zoneout), and returns a new cell.
-
- After applying modifiers the base cell should
- no longer be called directly. The modifer cell
- should be used instead.
- """
- def __init__(self, base_cell):
- super(ModifierCell, self).__init__()
- base_cell._modified = True
- self.base_cell = base_cell
-
- @property
- def params(self):
- self._own_params = False
- return self.base_cell.params
-
- @property
- def state_info(self):
- return self.base_cell.state_info
-
- def begin_state(self, init_sym=symbol.zeros, **kwargs): # pylint: disable=arguments-differ
- assert not self._modified, \
- "After applying modifier cells (e.g. DropoutCell) the base " \
- "cell cannot be called directly. Call the modifier cell instead."
- self.base_cell._modified = False
- begin = self.base_cell.begin_state(init_sym, **kwargs)
- self.base_cell._modified = True
- return begin
-
- def unpack_weights(self, args):
- return self.base_cell.unpack_weights(args)
-
- def pack_weights(self, args):
- return self.base_cell.pack_weights(args)
-
- def __call__(self, inputs, states):
- raise NotImplementedError
-
-
-class ZoneoutCell(ModifierCell):
- """Apply Zoneout on base cell.
-
- Parameters
- ----------
- base_cell : BaseRNNCell
- Cell on whose states to perform zoneout.
- zoneout_outputs : float, default 0.
- Fraction of the output that gets dropped out during training time.
- zoneout_states : float, default 0.
- Fraction of the states that gets dropped out during training time.
- """
- def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.):
- assert not isinstance(base_cell, FusedRNNCell), \
- "FusedRNNCell doesn't support zoneout. " \
- "Please unfuse first."
- assert not isinstance(base_cell, BidirectionalCell), \
- "BidirectionalCell doesn't support zoneout since it doesn't support step. " \
- "Please add ZoneoutCell to the cells underneath instead."
- assert not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \
- "Bidirectional SequentialRNNCell doesn't support zoneout. " \
- "Please add ZoneoutCell to the cells underneath instead."
- super(ZoneoutCell, self).__init__(base_cell)
- self.zoneout_outputs = zoneout_outputs
- self.zoneout_states = zoneout_states
- self.prev_output = None
-
- def reset(self):
- super(ZoneoutCell, self).reset()
- self.prev_output = None
-
- def __call__(self, inputs, states):
- cell, p_outputs, p_states = self.base_cell, self.zoneout_outputs, self.zoneout_states
- next_output, next_states = cell(inputs, states)
- mask = lambda p, like: symbol.Dropout(symbol.ones_like(like), p=p)
-
- prev_output = self.prev_output if self.prev_output is not None else symbol.zeros((0, 0))
-
- output = (symbol.where(mask(p_outputs, next_output), next_output, prev_output)
- if p_outputs != 0. else next_output)
- states = ([symbol.where(mask(p_states, new_s), new_s, old_s) for new_s, old_s in
- zip(next_states, states)] if p_states != 0. else next_states)
-
- self.prev_output = output
-
- return output, states
-
-
-class ResidualCell(ModifierCell):
- """Adds residual connection as described in Wu et al, 2016
- (https://arxiv.org/abs/1609.08144).
-
- Output of the cell is output of the base cell plus input.
-
- Parameters
- ----------
- base_cell : BaseRNNCell
- Cell on whose outputs to add residual connection.
- """
-
- def __init__(self, base_cell):
- super(ResidualCell, self).__init__(base_cell)
-
- def __call__(self, inputs, states):
- output, states = self.base_cell(inputs, states)
- output = symbol.elemwise_add(output, inputs, name="%s_plus_residual" % output.name)
- return output, states
-
- def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
- self.reset()
-
- self.base_cell._modified = False
- outputs, states = self.base_cell.unroll(length, inputs=inputs, begin_state=begin_state,
- layout=layout, merge_outputs=merge_outputs)
- self.base_cell._modified = True
-
- merge_outputs = isinstance(outputs, symbol.Symbol) if merge_outputs is None else \
- merge_outputs
- inputs, _ = _normalize_sequence(length, inputs, layout, merge_outputs)
- if merge_outputs:
- outputs = symbol.elemwise_add(outputs, inputs, name="%s_plus_residual" % outputs.name)
- else:
- outputs = [symbol.elemwise_add(output_sym, input_sym,
- name="%s_plus_residual" % output_sym.name)
- for output_sym, input_sym in zip(outputs, inputs)]
-
- return outputs, states
-
-
-class BidirectionalCell(BaseRNNCell):
- """Bidirectional RNN cell.
-
- Parameters
- ----------
- l_cell : BaseRNNCell
- cell for forward unrolling
- r_cell : BaseRNNCell
- cell for backward unrolling
- params : RNNParams, default None.
- Container for weight sharing between cells.
- A new RNNParams container is created if `params` is None.
- output_prefix : str, default ``'bi_'``
- prefix for name of output
- """
- def __init__(self, l_cell, r_cell, params=None, output_prefix='bi_'):
- super(BidirectionalCell, self).__init__('', params=params)
- self._output_prefix = output_prefix
- self._override_cell_params = params is not None
-
- if self._override_cell_params:
- assert l_cell._own_params and r_cell._own_params, \
- "Either specify params for BidirectionalCell " \
- "or child cells, not both."
- l_cell.params._params.update(self.params._params)
- r_cell.params._params.update(self.params._params)
- self.params._params.update(l_cell.params._params)
- self.params._params.update(r_cell.params._params)
- self._cells = [l_cell, r_cell]
-
- def unpack_weights(self, args):
- return _cells_unpack_weights(self._cells, args)
-
- def pack_weights(self, args):
- return _cells_pack_weights(self._cells, args)
-
- def __call__(self, inputs, states):
- raise NotImplementedError("Bidirectional cannot be stepped. Please use unroll")
-
- @property
- def state_info(self):
- return _cells_state_info(self._cells)
-
- def begin_state(self, **kwargs): # pylint: disable=arguments-differ
- assert not self._modified, \
- "After applying modifier cells (e.g. DropoutCell) the base " \
- "cell cannot be called directly. Call the modifier cell instead."
- return _cells_begin_state(self._cells, **kwargs)
-
- def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
- self.reset()
-
- inputs, axis = _normalize_sequence(length, inputs, layout, False)
- if begin_state is None:
- begin_state = self.begin_state()
-
- states = begin_state
- l_cell, r_cell = self._cells
- l_outputs, l_states = l_cell.unroll(length, inputs=inputs,
- begin_state=states[:len(l_cell.state_info)],
- layout=layout, merge_outputs=merge_outputs)
- r_outputs, r_states = r_cell.unroll(length,
- inputs=list(reversed(inputs)),
- begin_state=states[len(l_cell.state_info):],
- layout=layout, merge_outputs=merge_outputs)
-
- if merge_outputs is None:
- merge_outputs = (isinstance(l_outputs, symbol.Symbol)
- and isinstance(r_outputs, symbol.Symbol))
- if not merge_outputs:
- if isinstance(l_outputs, symbol.Symbol):
- l_outputs = list(symbol.SliceChannel(l_outputs, axis=axis,
- num_outputs=length, squeeze_axis=1))
- if isinstance(r_outputs, symbol.Symbol):
- r_outputs = list(symbol.SliceChannel(r_outputs, axis=axis,
- num_outputs=length, squeeze_axis=1))
-
- if merge_outputs:
- l_outputs = [l_outputs]
- r_outputs = [symbol.reverse(r_outputs, axis=axis)]
- else:
- r_outputs = list(reversed(r_outputs))
-
- outputs = [symbol.Concat(l_o, r_o, dim=1+merge_outputs,
- name=('%sout'%(self._output_prefix) if merge_outputs
- else '%st%d'%(self._output_prefix, i)))
- for i, l_o, r_o in
- zip(range(len(l_outputs)), l_outputs, r_outputs)]
-
- if merge_outputs:
- outputs = outputs[0]
-
- states = [l_states, r_states]
- return outputs, states
-
-
-class BaseConvRNNCell(BaseRNNCell):
- """Abstract base class for Convolutional RNN cells"""
- def __init__(self, input_shape, num_hidden,
- h2h_kernel, h2h_dilate,
- i2h_kernel, i2h_stride,
- i2h_pad, i2h_dilate,
- i2h_weight_initializer, h2h_weight_initializer,
- i2h_bias_initializer, h2h_bias_initializer,
- activation, prefix='', params=None, conv_layout='NCHW'):
- super(BaseConvRNNCell, self).__init__(prefix=prefix, params=params)
- # Convolution setting
- self._h2h_kernel = h2h_kernel
- assert (self._h2h_kernel[0] % 2 == 1) and (self._h2h_kernel[1] % 2 == 1), \
- "Only support odd number, get h2h_kernel= %s" % str(h2h_kernel)
- self._h2h_pad = (h2h_dilate[0] * (h2h_kernel[0] - 1) // 2,
- h2h_dilate[1] * (h2h_kernel[1] - 1) // 2)
- self._h2h_dilate = h2h_dilate
- self._i2h_kernel = i2h_kernel
- self._i2h_stride = i2h_stride
- self._i2h_pad = i2h_pad
- self._i2h_dilate = i2h_dilate
-
- self._num_hidden = num_hidden
- self._input_shape = input_shape
- self._conv_layout = conv_layout
- self._activation = activation
-
- # Infer state shape
- data = symbol.Variable('data')
- self._state_shape = symbol.Convolution(data=data,
- num_filter=self._num_hidden,
- kernel=self._i2h_kernel,
- stride=self._i2h_stride,
- pad=self._i2h_pad,
- dilate=self._i2h_dilate,
- layout=conv_layout)
- self._state_shape = self._state_shape.infer_shape(data=input_shape)[1][0]
- self._state_shape = (0, ) + self._state_shape[1:]
-
- # Get params
- self._iW = self.params.get('i2h_weight', init=i2h_weight_initializer)
- self._hW = self.params.get('h2h_weight', init=h2h_weight_initializer)
- self._iB = self.params.get('i2h_bias', init=i2h_bias_initializer)
- self._hB = self.params.get('h2h_bias', init=h2h_bias_initializer)
-
- @property
- def _num_gates(self):
- return len(self._gate_names)
-
- @property
- def state_info(self):
- return [{'shape': self._state_shape, '__layout__': self._conv_layout},
- {'shape': self._state_shape, '__layout__': self._conv_layout}]
-
- def _conv_forward(self, inputs, states, name):
-
- i2h = symbol.Convolution(name='%si2h'%name,
- data=inputs,
- num_filter=self._num_hidden*self._num_gates,
- kernel=self._i2h_kernel,
- stride=self._i2h_stride,
- pad=self._i2h_pad,
- dilate=self._i2h_dilate,
- weight=self._iW,
- bias=self._iB,
- layout=self._conv_layout)
-
- h2h = symbol.Convolution(name='%sh2h'%name,
- data=states[0],
- num_filter=self._num_hidden*self._num_gates,
- kernel=self._h2h_kernel,
- dilate=self._h2h_dilate,
- pad=self._h2h_pad,
- stride=(1, 1),
- weight=self._hW,
- bias=self._hB,
- layout=self._conv_layout)
- return i2h, h2h
-
- def __call__(self, inputs, states):
- raise NotImplementedError("BaseConvRNNCell is abstract class for convolutional RNN")
-
-class ConvRNNCell(BaseConvRNNCell):
- """Convolutional RNN cells
-
- Parameters
- ----------
- input_shape : tuple of int
- Shape of input in single timestep.
- num_hidden : int
- Number of units in output symbol.
- h2h_kernel : tuple of int, default (3, 3)
- Kernel of Convolution operator in state-to-state transitions.
- h2h_dilate : tuple of int, default (1, 1)
- Dilation of Convolution operator in state-to-state transitions.
- i2h_kernel : tuple of int, default (3, 3)
- Kernel of Convolution operator in input-to-state transitions.
- i2h_stride : tuple of int, default (1, 1)
- Stride of Convolution operator in input-to-state transitions.
- i2h_pad : tuple of int, default (1, 1)
- Pad of Convolution operator in input-to-state transitions.
- i2h_dilate : tuple of int, default (1, 1)
- Dilation of Convolution operator in input-to-state transitions.
- i2h_weight_initializer : str or Initializer
- Initializer for the input weights matrix, used for the convolution
- transformation of the inputs.
- h2h_weight_initializer : str or Initializer
- Initializer for the recurrent weights matrix, used for the convolution
- transformation of the recurrent state.
- i2h_bias_initializer : str or Initializer, default zeros
- Initializer for the bias vector.
- h2h_bias_initializer : str or Initializer, default zeros
- Initializer for the bias vector.
- activation : str or Symbol,
- default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
- Type of activation function.
- prefix : str, default ``'ConvRNN_'``
- Prefix for name of layers (and name of weight if params is None).
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- conv_layout : str, , default 'NCHW'
- Layout of ConvolutionOp
- """
- def __init__(self, input_shape, num_hidden,
- h2h_kernel=(3, 3), h2h_dilate=(1, 1),
- i2h_kernel=(3, 3), i2h_stride=(1, 1),
- i2h_pad=(1, 1), i2h_dilate=(1, 1),
- i2h_weight_initializer=None, h2h_weight_initializer=None,
- i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',
- activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2),
- prefix='ConvRNN_', params=None, conv_layout='NCHW'):
- super(ConvRNNCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden,
- h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,
- i2h_kernel=i2h_kernel, i2h_stride=i2h_stride,
- i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,
- i2h_weight_initializer=i2h_weight_initializer,
- h2h_weight_initializer=h2h_weight_initializer,
- i2h_bias_initializer=i2h_bias_initializer,
- h2h_bias_initializer=h2h_bias_initializer,
- activation=activation, prefix=prefix,
- params=params, conv_layout=conv_layout)
-
- @property
- def _gate_names(self):
- return ('',)
-
- def __call__(self, inputs, states):
- self._counter += 1
- name = '%st%d_'%(self._prefix, self._counter)
- i2h, h2h = self._conv_forward(inputs, states, name)
- output = self._get_activation(i2h + h2h, self._activation,
- name='%sout'%name)
- return output, [output]
-
- @property
- def state_info(self):
- return [{'shape': self._state_shape, '__layout__': self._conv_layout}]
-
-
-class ConvLSTMCell(BaseConvRNNCell):
- """Convolutional LSTM network cell.
-
- References
- ----------
- Xingjian et al. NIPS2015
-
- Parameters
- ----------
- input_shape : tuple of int
- Shape of input in single timestep.
- num_hidden : int
- Number of units in output symbol.
- h2h_kernel : tuple of int, default (3, 3)
- Kernel of Convolution operator in state-to-state transitions.
- h2h_dilate : tuple of int, default (1, 1)
- Dilation of Convolution operator in state-to-state transitions.
- i2h_kernel : tuple of int, default (3, 3)
- Kernel of Convolution operator in input-to-state transitions.
- i2h_stride : tuple of int, default (1, 1)
- Stride of Convolution operator in input-to-state transitions.
- i2h_pad : tuple of int, default (1, 1)
- Pad of Convolution operator in input-to-state transitions.
- i2h_dilate : tuple of int, default (1, 1)
- Dilation of Convolution operator in input-to-state transitions.
- i2h_weight_initializer : str or Initializer
- Initializer for the input weights matrix, used for the convolution
- transformation of the inputs.
- h2h_weight_initializer : str or Initializer
- Initializer for the recurrent weights matrix, used for the convolution
- transformation of the recurrent state.
- i2h_bias_initializer : str or Initializer, default zeros
- Initializer for the bias vector.
- h2h_bias_initializer : str or Initializer, default zeros
- Initializer for the bias vector.
- activation : str or Symbol
- default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
- Type of activation function.
- prefix : str, default ``'ConvLSTM_'``
- Prefix for name of layers (and name of weight if params is None).
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- conv_layout : str, , default 'NCHW'
- Layout of ConvolutionOp
- """
- def __init__(self, input_shape, num_hidden,
- h2h_kernel=(3, 3), h2h_dilate=(1, 1),
- i2h_kernel=(3, 3), i2h_stride=(1, 1),
- i2h_pad=(1, 1), i2h_dilate=(1, 1),
- i2h_weight_initializer=None, h2h_weight_initializer=None,
- i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',
- activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2),
- prefix='ConvLSTM_', params=None,
- conv_layout='NCHW'):
- super(ConvLSTMCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden,
- h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,
- i2h_kernel=i2h_kernel, i2h_stride=i2h_stride,
- i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,
- i2h_weight_initializer=i2h_weight_initializer,
- h2h_weight_initializer=h2h_weight_initializer,
- i2h_bias_initializer=i2h_bias_initializer,
- h2h_bias_initializer=h2h_bias_initializer,
- activation=activation, prefix=prefix,
- params=params, conv_layout=conv_layout)
-
- @property
- def _gate_names(self):
- return ['_i', '_f', '_c', '_o']
-
- def __call__(self, inputs, states):
- self._counter += 1
- name = '%st%d_'%(self._prefix, self._counter)
- i2h, h2h = self._conv_forward(inputs, states, name)
- gates = i2h + h2h
- slice_gates = symbol.SliceChannel(gates, num_outputs=4, axis=self._conv_layout.find('C'),
- name="%sslice"%name)
- in_gate = symbol.Activation(slice_gates[0], act_type="sigmoid",
- name='%si'%name)
- forget_gate = symbol.Activation(slice_gates[1], act_type="sigmoid",
- name='%sf'%name)
- in_transform = self._get_activation(slice_gates[2], self._activation,
- name='%sc'%name)
- out_gate = symbol.Activation(slice_gates[3], act_type="sigmoid",
- name='%so'%name)
- next_c = symbol._internal._plus(forget_gate * states[1], in_gate * in_transform,
- name='%sstate'%name)
- next_h = symbol._internal._mul(out_gate, self._get_activation(next_c, self._activation),
- name='%sout'%name)
-
- return next_h, [next_h, next_c]
-
- @property
- def state_info(self):
- return [{'shape': self._state_shape, '__layout__': self._conv_layout},
- {'shape': self._state_shape, '__layout__': self._conv_layout}]
-
-class ConvGRUCell(BaseConvRNNCell):
- """Convolutional Gated Rectified Unit (GRU) network cell.
-
- Parameters
- ----------
- input_shape : tuple of int
- Shape of input in single timestep.
- num_hidden : int
- Number of units in output symbol.
- h2h_kernel : tuple of int, default (3, 3)
- Kernel of Convolution operator in state-to-state transitions.
- h2h_dilate : tuple of int, default (1, 1)
- Dilation of Convolution operator in state-to-state transitions.
- i2h_kernel : tuple of int, default (3, 3)
- Kernel of Convolution operator in input-to-state transitions.
- i2h_stride : tuple of int, default (1, 1)
- Stride of Convolution operator in input-to-state transitions.
- i2h_pad : tuple of int, default (1, 1)
- Pad of Convolution operator in input-to-state transitions.
- i2h_dilate : tuple of int, default (1, 1)
- Dilation of Convolution operator in input-to-state transitions.
- i2h_weight_initializer : str or Initializer
- Initializer for the input weights matrix, used for the convolution
- transformation of the inputs.
- h2h_weight_initializer : str or Initializer
- Initializer for the recurrent weights matrix, used for the convolution
- transformation of the recurrent state.
- i2h_bias_initializer : str or Initializer, default zeros
- Initializer for the bias vector.
- h2h_bias_initializer : str or Initializer, default zeros
- Initializer for the bias vector.
- activation : str or Symbol,
- default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
- Type of activation function.
- prefix : str, default ``'ConvGRU_'``
- Prefix for name of layers (and name of weight if params is None).
- params : RNNParams, default None
- Container for weight sharing between cells. Created if None.
- conv_layout : str, , default 'NCHW'
- Layout of ConvolutionOp
- """
- def __init__(self, input_shape, num_hidden,
- h2h_kernel=(3, 3), h2h_dilate=(1, 1),
- i2h_kernel=(3, 3), i2h_stride=(1, 1),
- i2h_pad=(1, 1), i2h_dilate=(1, 1),
- i2h_weight_initializer=None, h2h_weight_initializer=None,
- i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',
- activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2),
- prefix='ConvGRU_', params=None, conv_layout='NCHW'):
- super(ConvGRUCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden,
- h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,
- i2h_kernel=i2h_kernel, i2h_stride=i2h_stride,
- i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,
- i2h_weight_initializer=i2h_weight_initializer,
- h2h_weight_initializer=h2h_weight_initializer,
- i2h_bias_initializer=i2h_bias_initializer,
- h2h_bias_initializer=h2h_bias_initializer,
- activation=activation, prefix=prefix,
- params=params, conv_layout=conv_layout)
-
- @property
- def _gate_names(self):
- return ['_r', '_z', '_o']
-
- @property
- def state_info(self):
- return [{'shape': self._state_shape, '__layout__': self._conv_layout}]
-
- def __call__(self, inputs, states):
- self._counter += 1
- seq_idx = self._counter
- name = '%st%d_' % (self._prefix, seq_idx)
- i2h, h2h = self._conv_forward(inputs, states, name)
- # pylint: disable=unbalanced-tuple-unpacking
- i2h_r, i2h_z, i2h = symbol.SliceChannel(i2h, num_outputs=3, name="%s_i2h_slice" % name)
- h2h_r, h2h_z, h2h = symbol.SliceChannel(h2h, num_outputs=3, name="%s_h2h_slice" % name)
-
- reset_gate = symbol.Activation(i2h_r + h2h_r, act_type="sigmoid",
- name="%s_r_act" % name)
- update_gate = symbol.Activation(i2h_z + h2h_z, act_type="sigmoid",
- name="%s_z_act" % name)
-
- next_h_tmp = self._get_activation(i2h + reset_gate * h2h, self._activation,
- name="%s_h_act" % name)
-
- next_h = symbol._internal._plus((1. - update_gate) * next_h_tmp, update_gate * states[0],
- name='%sout' % name)
-
- return next_h, [next_h]
diff --git a/tests/python/gpu/test_contrib_amp.py b/tests/python/gpu/test_contrib_amp.py
index 4d943bd..d1d9162 100644
--- a/tests/python/gpu/test_contrib_amp.py
+++ b/tests/python/gpu/test_contrib_amp.py
@@ -33,7 +33,6 @@ curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
sys.path.insert(0, os.path.join(curr_path, '../unittest'))
from common import with_seed, teardown_module, assert_raises_cudnn_not_satisfied
sys.path.insert(0, os.path.join(curr_path, '../train'))
-from test_bucketing import train_model
set_default_context(mx.gpu(0))
@pytest.fixture()
@@ -314,44 +313,10 @@ def test_amp_conversion(amp_tests):
assert params["stage2_unit1_conv2_weight"].dtype == np.float16
- def check_amp_convert_bucketing_module():
- model = train_model(context=mx.current_context())
- result_model = amp.convert_bucketing_module(model)
- val_sent = []
- batch_size = 128
- invalid_label = -1
- num_sentence = 1000
- buckets = [5, 10, 20, 30, 40]
- len_vocab = 50
-
- for _ in range(num_sentence):
- len_sentence = randint(6, max(buckets)-1) # leave out the two last buckets empty
- val_sentence = []
- for _ in range(len_sentence):
- val_sentence.append(randint(1, len_vocab))
- val_sent.append(val_sentence)
-
- data_val = mx.rnn.BucketSentenceIter(val_sent, batch_size, buckets=buckets,
- invalid_label=invalid_label)
- result_model.bind(data_val.provide_data, data_val.provide_label, for_training=False)
- result_model.score(data_val, mx.gluon.metric.Perplexity(invalid_label),
- batch_end_callback=mx.callback.Speedometer(batch_size, 1))
-
- # AMP conversion with cast_optional_params set to true
- # Flaky test when cast_optional_params set to True : https://github.com/apache/incubator-mxnet/issues/16030
- '''
- result_model = amp.convert_bucketing_module(model, cast_optional_params=True)
- result_model.bind(data_val.provide_data, data_val.provide_label, for_training=False)
- result_model.score(data_val, mx.gluon.metric.Perplexity(invalid_label),
- batch_end_callback=mx.callback.Speedometer(batch_size, 1))
- '''
-
-
with mx.Context(mx.gpu(0)):
check_amp_convert_symbol()
check_amp_convert_model()
check_amp_convert_hybrid_block()
- check_amp_convert_bucketing_module()
@with_seed()
@pytest.mark.skip(reason='Error during waitall(). Tracked in #18099')
diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py
index e92c296..b6d0011 100644
--- a/tests/python/gpu/test_operator_gpu.py
+++ b/tests/python/gpu/test_operator_gpu.py
@@ -38,7 +38,6 @@ from test_numpy_interoperability import *
from test_optimizer import *
from test_random import *
from test_exc_handling import *
-#from test_rnn import *
from test_sparse_ndarray import *
from test_sparse_operator import *
from test_ndarray import *
@@ -1742,90 +1741,6 @@ def check_rnn_consistency(cell1, cell2):
mx.test_utils.assert_allclose(mod1.get_outputs()[0], mod2.get_outputs()[0], rtol=1e-2, atol=1e-4)
@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnn():
- fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='rnn_relu', prefix='')
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.RNNCell(100, activation='relu', prefix='l0_'))
- stack.add(mx.rnn.RNNCell(100, activation='relu', prefix='l1_'))
-
- check_rnn_consistency(fused, stack)
- check_rnn_consistency(stack, fused)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-def test_lstm_forget_bias():
- forget_bias = 2.0
- fused = mx.rnn.FusedRNNCell(10, forget_bias=forget_bias, num_layers=2, mode='lstm', prefix='')
-
- dshape = (32, 1, 20)
- data = mx.sym.Variable('data')
-
- sym, _ = fused.unroll(1, data, merge_outputs=True)
- mod = mx.mod.Module(sym, label_names=None, context=mx.gpu(0))
- mod.bind(data_shapes=[('data', dshape)], label_shapes=None)
-
- mod.init_params()
-
- args, auxs = mod.get_params()
- args = fused.unpack_weights(args)
-
- bias_name = next(x for x in args if x.endswith('f_bias'))
- expected_bias = forget_bias * np.ones(10, )
- mx.test_utils.assert_allclose(args[bias_name], expected_bias)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_gru():
- fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='')
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.GRUCell(100, prefix='l0_'))
- stack.add(mx.rnn.GRUCell(100, prefix='l1_'))
-
- check_rnn_consistency(fused, stack)
- check_rnn_consistency(stack, fused)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_bidirectional():
- fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='',
- bidirectional=True)
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.GRUCell(100, prefix='l0_'),
- mx.rnn.GRUCell(100, prefix='r0_'),
- output_prefix='bi_gru_0_'))
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.GRUCell(100, prefix='l1_'),
- mx.rnn.GRUCell(100, prefix='r1_'),
- output_prefix='bi_gru_1_'))
-
- check_rnn_consistency(fused, stack)
- check_rnn_consistency(stack, fused)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-def test_unfuse():
- for mode in ['rnn_tanh', 'rnn_relu', 'lstm', 'gru']:
- fused = mx.rnn.FusedRNNCell(
- 100, num_layers=2, mode=mode,
- prefix='test_%s'%mode,
- bidirectional=True,
- dropout=0.5)
-
- stack = fused.unfuse()
-
- check_rnn_consistency(fused, stack)
- check_rnn_consistency(stack, fused)
-
-
-@with_seed()
@pytest.mark.serial
def test_psroipooling_with_type():
arg_params = {
@@ -2036,29 +1951,6 @@ def test_deformable_convolution_options():
check_consistency(sym, ctx_list, scale=0.1, tol=tol)
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_residual_fused():
- cell = mx.rnn.ResidualCell(
- mx.rnn.FusedRNNCell(50, num_layers=3, mode='lstm',
- prefix='rnn_', dropout=0.5))
-
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)]
- outputs, _ = cell.unroll(2, inputs, merge_outputs=None)
- assert sorted(cell.params._params.keys()) == \
- ['rnn_parameters']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50))
- assert outs == [(10, 2, 50)]
- outputs = outputs.eval(ctx=mx.gpu(0),
- rnn_t0_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5,
- rnn_t1_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5,
- rnn_parameters=mx.nd.zeros((61200,), ctx=mx.gpu(0)))
- expected_outputs = np.ones((10, 2, 50))+5
- assert np.array_equal(outputs[0].asnumpy(), expected_outputs)
-
-
def check_rnn_layer(layer):
layer.collect_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)])
with mx.gpu(0):
diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py
index 71f044f..5034d90 100644
--- a/tests/python/quantization/test_quantization.py
+++ b/tests/python/quantization/test_quantization.py
@@ -957,169 +957,6 @@ def test_quantize_model():
for qdtype in ['int8', 'uint8']:
check_quantize_model(qdtype)
-@with_seed()
-def test_quantize_model_with_forward():
- def check_quantize_model(qdtype):
- if is_test_for_native_cpu():
- print('skipped testing test_quantize_model_with_forward for native cpu since it is not supported yet')
- return
- elif qdtype == 'uint8' and is_test_for_gpu():
- print('skipped testing test_quantize_model_with_forward for gpu uint8 since it is not supported yet')
- return
-
- def check_params(params, qparams, qsym=None):
- if qsym is None:
- assert len(params) == len(qparams)
- for k, v in params.items():
- assert k in qparams
- assert same(v.asnumpy(), qparams[k].asnumpy())
- else:
- qparams_ground_truth = mx.contrib.quant._quantize_params(qsym, params, th_dict = {})
- assert len(qparams) == len(qparams_ground_truth)
- for k, v in qparams_ground_truth.items():
- assert k in qparams
- assert same(v.asnumpy(), qparams[k].asnumpy())
-
- def check_qsym_calibrated(qsym):
- attrs = qsym.attr_dict()
- for k, v in attrs.items():
- if k.find('requantize_') != -1:
- assert 'min_calib_range' in v
- assert 'max_calib_range' in v
-
- def check_qsym_qdtype(qsym, qdtype):
- attrs = qsym.attr_dict()
- for k, v in attrs.items():
- if k.find('_quantize') != -1:
- assert 'out_type' in v
- assert v['out_type'] == qdtype
-
- def check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape=None):
- if label_shape is None:
- mod = mx.mod.Module(symbol=qsym, label_names=None, context=mx.current_context())
- mod.bind(for_training=False,
- data_shapes=[('data', data_shape)])
- else:
- mod = mx.mod.Module(symbol=qsym, context=mx.current_context())
- mod.bind(for_training=False,
- data_shapes=[('data', data_shape)],
- label_shapes=[('softmax_label', label_shape)])
- mod.set_params(qarg_params, qaux_params)
- data = [mx.random.uniform(-1.0, 1.0, shape=shape) for _, shape in mod.data_shapes]
- batch = mx.io.DataBatch(data, [])
- mod.forward(batch, is_train=False)
- for output in mod.get_outputs():
- output.wait_to_read()
-
- batch_size = 4
- length = batch_size # specify num of outputs from split op
- sym_list = []
- name_list = []
- dshape_list = []
- lshape_list = []
-
- # sym 1
- sym_list.append(get_fp32_residual())
- name_list.append('sym1')
- dshape_list.append((batch_size, 4, 10, 10))
- lshape_list.append((batch_size, 10))
-
- # sym 2
- sym_list.append(get_fp32_sym_with_multiple_outputs(length))
- name_list.append('sym2')
- dshape_list.append((length, 4, 4, 10, 10))
- lshape_list.append((length, 10))
-
- data = mx.sym.Variable('data')
- # sym 3
- sym_list.append(mx.sym.Convolution(data, kernel=(1, 1), num_filter=16, name='conv0'))
- name_list.append('sym3')
- dshape_list.append((batch_size, 4, 10, 10))
- lshape_list.append(None)
-
- # sym 4
- cell = mx.rnn.LSTMCell(num_hidden=64)
- outputs, _ = cell.unroll(length, data)
- sym_list.append(mx.sym.Group(outputs))
- name_list.append('sym4')
- dshape_list.append((batch_size, length, 32))
- lshape_list.append(None)
-
- for s, dshape, lshape, name in zip(sym_list, dshape_list, lshape_list, name_list):
- if qdtype == 'int8' and name in ['sym1','sym2','sym3']:
- print('mkldnn_quantized_conv op only supports uint8 as input type, skip test with int8.')
- continue
- if qdtype == 'uint8' and name in ['sym1']:
- print('mkldnn_quantized_bn doesn\'t support calib_mode=None')
- continue
- if lshape is None:
- mod = Module(symbol=s, label_names=None)
- mod.bind(for_training=False,
- data_shapes=[('data', dshape)])
- else:
- mod = Module(symbol=s)
- mod.bind(for_training=False,
- data_shapes=[('data', dshape)],
- label_shapes=[('softmax_label', lshape)])
-
- mod.init_params()
- arg_params, aux_params = mod.get_params()
-
- excluded_sym_names = []
- excluded_op_names = []
- # sym3/sym4 doesn't have such layers
- if name not in ['sym3', 'sym4']:
- excluded_names = []
- if mx.current_context() == mx.cpu():
- excluded_op_names += ['FullyConnected']
- excluded_names += ['conv1']
- excluded_names += ['concat']
-
- optional_names = ['pool0']
- for skip_optional_names in [False, True]:
- exclude_sym_names = []
- if skip_optional_names:
- excluded_sym_names = excluded_names
- else:
- excluded_sym_names = excluded_names + optional_names
- if name == 'sym4':
- excluded_op_names += ['elemwise_add', 'elemwise_mul']
-
- qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=s,
- arg_params=arg_params,
- aux_params=aux_params,
- excluded_sym_names=excluded_sym_names,
- excluded_op_names=excluded_op_names,
- ctx=mx.current_context(),
- quantized_dtype=qdtype,
- calib_mode='none',
- quantize_mode='full')
- check_params(arg_params, qarg_params, qsym)
- check_params(aux_params, qaux_params)
- check_qsym_forward(qsym, qarg_params, qaux_params, dshape, lshape)
-
- calib_data = mx.nd.random.uniform(shape=dshape)
- calib_data = NDArrayIter(data=calib_data, batch_size=batch_size)
- calib_data = DummyIter(calib_data)
- qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=s,
- arg_params=arg_params,
- aux_params=aux_params,
- excluded_sym_names=excluded_sym_names,
- excluded_op_names=excluded_op_names,
- ctx=mx.current_context(),
- quantized_dtype=qdtype,
- calib_mode='naive',
- calib_data=calib_data,
- num_calib_examples=20,
- quantize_mode='full')
- check_params(arg_params, qarg_params, qsym)
- check_params(aux_params, qaux_params)
- check_qsym_calibrated(qsym)
- check_qsym_qdtype(qsym, qdtype)
- check_qsym_forward(qsym, qarg_params, qaux_params, dshape, lshape)
-
- for qdtype in ['int8', 'uint8']:
- check_quantize_model(qdtype)
@with_seed()
def test_quantize_gluon_with_forward():
diff --git a/tests/python/train/test_bucketing.py b/tests/python/train/test_bucketing.py
deleted file mode 100644
index 6734a5d..0000000
--- a/tests/python/train/test_bucketing.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# pylint: skip-file
-import numpy as np
-import mxnet as mx
-import random
-from random import randint
-from mxnet.contrib.amp import amp
-
-import pytest
-
-
-def prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence):
- train_sent = []
- val_sent = []
-
- for _ in range(num_sentence):
- len_sentence = randint(6, max(buckets)-1) # leave out the two last buckets empty
- train_sentence = []
- val_sentence = []
- for _ in range(len_sentence):
- train_sentence.append(randint(1, len_vocab))
- val_sentence.append(randint(1, len_vocab))
- train_sent.append(train_sentence)
- val_sent.append(val_sentence)
-
- data_train = mx.rnn.BucketSentenceIter(train_sent, batch_size, buckets=buckets,
- invalid_label=invalid_label)
- data_val = mx.rnn.BucketSentenceIter(val_sent, batch_size, buckets=buckets,
- invalid_label=invalid_label)
-
- return (data_train, data_val)
-
-
-def train_model(context=mx.cpu()):
- import logging
- head = '%(asctime)-15s %(message)s'
- logging.basicConfig(level=logging.DEBUG, format=head)
- console = logging.StreamHandler()
- console.setLevel(logging.DEBUG)
- logging.getLogger('').addHandler(console)
-
- batch_size = 128
- num_epochs = 5
- num_hidden = 25
- num_embed = 25
- num_layers = 2
- len_vocab = 50
- buckets = [5, 10, 20, 30, 40]
-
- invalid_label = -1
- num_sentence = 1000
-
- data_train, data_val = prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence)
-
- stack = mx.rnn.SequentialRNNCell()
- for i in range(num_layers):
- stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i))
-
- def sym_gen(seq_len):
- data = mx.sym.Variable('data')
- label = mx.sym.Variable('softmax_label')
- embed = mx.sym.Embedding(data=data, input_dim=len_vocab,
- output_dim=num_embed, name='embed')
-
- stack.reset()
- outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
- pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
- pred = mx.sym.FullyConnected(data=pred, num_hidden=len_vocab, name='pred')
-
- label = mx.sym.Reshape(label, shape=(-1,))
- loss = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-
- return loss, ('data',), ('softmax_label',)
-
- contexts = context
-
- model = mx.mod.BucketingModule(
- sym_gen=sym_gen,
- default_bucket_key=data_train.default_bucket_key,
- context=contexts)
-
- logging.info('Begin fit...')
- model.fit(
- train_data=data_train,
- eval_data=data_val,
- eval_metric=mx.gluon.metric.Perplexity(invalid_label), # Use Perplexity for multiclass classification.
- kvstore='device',
- optimizer='sgd',
- optimizer_params={'learning_rate': 0.01,
- 'momentum': 0,
- 'wd': 0.00001},
- initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
- num_epoch=num_epochs,
- batch_end_callback=mx.callback.Speedometer(batch_size, 50))
- logging.info('Finished fit...')
- return model
-
-
-@pytest.mark.garbage_expected
-def test_bucket_module():
- # This test forecasts random sequence of words to check bucketing.
- # We cannot guarantee the accuracy of such an impossible task, and comments out the following line.
- # assert model.score(data_val, mx.gluon.metric.MSE())[0][1] < 350, "High mean square error."
- model = train_model()
-
-
-if __name__ == "__main__":
- test_bucket_module()
diff --git a/tests/python/unittest/test_contrib_control_flow.py b/tests/python/unittest/test_contrib_control_flow.py
index b703402..3c8950d 100644
--- a/tests/python/unittest/test_contrib_control_flow.py
+++ b/tests/python/unittest/test_contrib_control_flow.py
@@ -899,101 +899,6 @@ def test_while_loop_nested():
assert_almost_equal(x, y, rtol=1e-3, atol=1e-3)
-@with_seed()
-def test_while_loop_rnn():
- def _array(shape):
- return mx.nd.random.uniform(-1.0, 1.0, shape=shape)
-
- cell_types = [mx.rnn.LSTMCell]
- num_params = [2]
-
- batch_size = 2
- hidden_dim = 3
- input_dim = 4
- seq_len = 3
-
- for cell, n_param in zip(cell_types, num_params):
- # using while_loop
- params = mx.rnn.RNNParams()
- data = mx.sym.var("data")
- iter_i = mx.sym.var("i")
- def _cond(*states):
- i = states[0]
- return i < seq_len
- def _func(*states):
- i = states[0]
- states = states[1:]
- in_ = data.take(i).squeeze(axis=0)
- rnn = cell(hidden_dim, prefix='', params=params)
- next_hidden, next_states = rnn(in_, states)
- return [next_hidden], [i + 1] + list(next_states)
- states = [mx.sym.var("s_" + str(i)) for i in range(n_param)]
- result = mx.sym.contrib.while_loop(
- cond=_cond,
- func=_func,
- loop_vars=[iter_i] + states,
- max_iterations=seq_len
- )
- result = mx.sym.Group(result[0] + result[1][1: ])
- rnn_inputs = result.list_inputs()
- args = {
- "i": mx.nd.zeros([1]),
- "data": _array((seq_len, batch_size, input_dim)),
- "i2h_weight": _array((input_dim * hidden_dim, input_dim)),
- "i2h_bias": _array((input_dim * hidden_dim, )),
- "s_0": _array((batch_size, hidden_dim)),
- "h2h_weight": _array((input_dim * hidden_dim, seq_len)),
- "h2h_bias": _array((input_dim * hidden_dim, )),
- "s_1": _array((batch_size, hidden_dim)),
- }
- args_grad = {
- "i": _array([1]),
- "data": _array((seq_len, batch_size, input_dim)),
- "i2h_weight": _array((input_dim * hidden_dim, input_dim)),
- "i2h_bias": _array((input_dim * hidden_dim, )),
- "s_0": _array((batch_size, hidden_dim)),
- "h2h_weight": _array((input_dim * hidden_dim, seq_len)),
- "h2h_bias": _array((input_dim * hidden_dim, )),
- "s_1": _array((batch_size, hidden_dim)),
- }
- e_1 = result.bind(ctx=default_context(),
- args={name: array.copy() for name, array in args.items()},
- args_grad={name: array.copy() for name, array in args_grad.items() if name != "i"},
- )
- # using unrolled rnn
- rnn = cell(hidden_dim, prefix='')
- unroll_outs = []
- for inputs in mx.sym.split(data, num_outputs=seq_len, axis=0, squeeze_axis=True):
- h, states = rnn(inputs, states)
- unroll_outs.append(mx.sym.expand_dims(h, axis=0))
- unroll_outs = _as_list(mx.sym.concat(*unroll_outs, dim=0))
- unroll_outs.extend(states)
- result = mx.sym.Group(unroll_outs)
- e_2 = result.bind(ctx=default_context(),
- args={name: array.copy() for name, array in args.items() if name != "i"},
- args_grad={name: array.copy() for name, array in args_grad.items() if name != "i"},
- )
- for case_id in range(100):
- args = {name: array.copy() for name, array in args.items()}
- e_1.forward(is_train=True, **args)
- out_grads = [_array(arr.shape) for arr in e_1.outputs]
- e_1.backward(out_grads)
- args = {name: array.copy() for name, array in args.items() if name != "i"}
- e_2.forward(is_train=True, **args)
- e_2.backward(out_grads)
- assert len(e_1.outputs) == len(e_2.outputs)
- for x, y in zip(e_1.outputs, e_2.outputs):
- x = x.asnumpy()
- y = y.asnumpy()
- assert_almost_equal(x, y, rtol=1e-3, atol=1e-3)
- grad_keys = list(e_2.grad_dict.keys())
- e_1_grad = [e_1.grad_dict[x] for x in grad_keys]
- e_2_grad = [e_2.grad_dict[x] for x in grad_keys]
- for x, y in zip(e_1_grad, e_2_grad):
- x = x.asnumpy()
- y = y.asnumpy()
- assert_almost_equal(x, y, rtol=1e-3, atol=1e-3)
-
def _verify_cond(cond_func, then_func, else_func, input_var_shapes, free_var_shapes, is_train):
def _create_symbol(prefix, i):
@@ -1586,104 +1491,6 @@ def test_foreach_nested():
assert_almost_equal(state.grad.asnumpy(), state_grad.asnumpy(), rtol=1e-3, atol=1e-3)
-def check_foreach_rnn(cell_type, num_states):
- data = mx.sym.var("data")
- params = mx.rnn.RNNParams()
- hidden_dim = 4
- input_dim = 5
- seq_len = 2
- batch_size = 2
-
- # This tests foreach with accumulation sum.
- def step(in1, states):
- rnn = cell_type(hidden_dim, prefix='', params=params)
- next_h, states = rnn(in1, states)
- return (next_h, states)
-
- def sym_group(out):
- if (isinstance(out[0], mx.sym.Symbol)):
- ret = [out[0]]
- else:
- ret = out[0]
- ret.extend(out[1])
- return mx.sym.Group(ret)
-
- rnn = cell_type(hidden_dim, prefix='', params=params)
- if num_states == 2:
- init_states = [mx.sym.var("h"), mx.sym.var("c")]
- else:
- init_states = [mx.sym.var("h")]
- out = mx.sym.contrib.foreach(step, data, init_states)
- out = sym_group(out)
- arg_shapes, out_shapes, aux_shapes = out.infer_shape(data=(seq_len, batch_size, input_dim),
- h=(batch_size, hidden_dim))
- rnn_inputs = out.list_inputs()
-
- # Inputs
- args1 = {name:mx.nd.random.uniform(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
- args2 = copy.deepcopy(args1)
- # gradients for the backward of the foreach symbol
- args_grad1 = {name:mx.nd.empty(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
- # gradients for the backward of the unrolled symbol.
- args_grad2 = {name:mx.nd.empty(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
-
- # Symbol of running LSTM with foreach.
- out = mx.sym.contrib.foreach(step, data, init_states)
- out = sym_group(out)
- js_1 = out.tojson()
- out = mx.sym.load_json(js_1)
- js_2 = out.tojson()
- assert js_1 == js_2
- e1 = out.bind(ctx=default_context(), args=args1, args_grad=args_grad1)
-
- # Symbol of running unrolled LSTM.
- lstm = cell_type(hidden_dim, prefix='')
- unroll_outs = []
- states = init_states
- for inputs in mx.sym.split(data, num_outputs=seq_len, axis=0, squeeze_axis=True):
- h, states = lstm(inputs, states)
- unroll_outs.append(mx.sym.expand_dims(h, axis=0))
- unroll_outs = _as_list(mx.sym.concat(*unroll_outs, dim=0))
- unroll_outs.extend(states)
- out = mx.sym.Group(unroll_outs)
- js_1 = out.tojson()
- out = mx.sym.load_json(js_1)
- js_2 = out.tojson()
- assert js_1 == js_2
- e2 = out.bind(ctx=default_context(), args=args2, args_grad=args_grad2)
-
- for i in range(5):
- out_grads = []
- for arr in e1.outputs:
- out_grads.append(mx.nd.random.uniform(-10, 10, arr.shape))
-
- args = {name:mx.nd.random.uniform(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
-
- e1.forward(is_train=True, **args)
- outputs1 = e1.outputs
- e1.backward(out_grads)
-
- e2.forward(is_train=True, **args)
- outputs2 = e2.outputs
- e2.backward(out_grads)
-
- for i in range(len(outputs2)):
- assert_almost_equal(outputs1[i].asnumpy(), outputs2[i].asnumpy(),
- rtol=1e-3, atol=1e-3)
- input_names = out.list_inputs()
- for i in range(len(e1.grad_arrays)):
- name = input_names[i]
- assert_almost_equal(args_grad1[name].asnumpy(), args_grad2[name].asnumpy(),
- rtol=1e-3, atol=1e-3)
-
-
-@with_seed()
-def test_foreach_rnn():
- cell_types = [(mx.rnn.LSTMCell, 2), (mx.rnn.RNNCell, 1), (mx.rnn.GRUCell, 1)]
- for cell_type, num_states in cell_types:
- check_foreach_rnn(cell_type, num_states)
-
-
@with_seed()
def test_cut_subgraph_foreach():
class TestLayer(gluon.HybridBlock):
diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py
index bfb72a4..b7f12ff 100644
--- a/tests/python/unittest/test_module.py
+++ b/tests/python/unittest/test_module.py
@@ -26,7 +26,6 @@ from common import setup_module, with_seed, assertRaises, teardown_module
from collections import namedtuple
curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
sys.path.insert(0, os.path.join(curr_path, "../train"))
-from test_bucketing import train_model, prepare_bucketing_data
import pytest
@@ -212,76 +211,6 @@ def test_save_load(ctx, get_updater, tmpdir):
@with_seed()
-@pytest.mark.garbage_expected
-def test_bucketing_save_load(tmpdir):
- previous_update_on_kvstore = os.getenv('MXNET_UPDATE_ON_KVSTORE', "1")
- os.putenv('MXNET_UPDATE_ON_KVSTORE', '1')
- def dict_equ(a, b):
- assert set(a) == set(b)
- for k in a:
- assert (a[k].asnumpy() == b[k].asnumpy()).all()
-
-
- len_vocab = 50
- num_embed = 25
- num_epochs = 5
- batch_size = 128
- num_layers = 2
- num_hidden = 25
- buckets = [5, 10, 20, 30, 40]
- invalid_label = -1
- num_sentence=1000
-
- stack = mx.rnn.SequentialRNNCell()
- for i in range(num_layers):
- stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i))
-
- def sym_gen(seq_len):
- data = mx.sym.Variable('data')
- label = mx.sym.Variable('softmax_label')
- embed = mx.sym.Embedding(data=data, input_dim=len_vocab,
- output_dim=num_embed, name='embed')
- stack.reset()
- outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
- pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
- pred = mx.sym.FullyConnected(data=pred, num_hidden=len_vocab, name='pred')
-
- label = mx.sym.Reshape(label, shape=(-1,))
- loss = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-
- return loss, ('data',), ('softmax_label',)
-
- path = str(tmpdir.join('test'))
-
- model = train_model(context=mx.current_context())
- model.save_checkpoint(path, 0)
- data_train, data_val = prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence)
- mod2 = mx.mod.BucketingModule.load(path, 0, sym_gen=sym_gen,
- default_bucket_key=data_train.default_bucket_key)
-
- mod2.bind(data_shapes=data_train.provide_data,
- label_shapes=data_train.provide_label)
-
- for bucket_key in model._buckets.keys():
- dict_equ(model._buckets[model._default_bucket_key].get_params()[0],
- mod2._buckets[mod2._default_bucket_key].get_params()[0])
- mod2.fit(
- train_data=data_train,
- eval_data=data_val,
- eval_metric=mx.gluon.metric.Perplexity(invalid_label), # Use Perplexity for multiclass classification.
- kvstore='device',
- optimizer='sgd',
- optimizer_params={'learning_rate': 0.01,
- 'momentum': 0,
- 'wd': 0.00001},
- initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
- num_epoch=num_epochs,
- batch_end_callback=mx.callback.Speedometer(batch_size, 50))
- os.putenv('MXNET_UPDATE_ON_KVSTORE', previous_update_on_kvstore)
-
-
-@with_seed()
def test_module_reshape():
data = mx.sym.Variable('data')
sym = mx.sym.FullyConnected(data, num_hidden=20, name='fc')
@@ -309,93 +238,6 @@ def test_module_reshape():
assert (mod.get_params()[0]['fc_bias'].asnumpy() == -3).all()
-@with_seed()
-def test_module_states():
- stack = mx.rnn.SequentialRNNCell()
- for i in range(2):
- stack.add(mx.rnn.LSTMCell(num_hidden=20, prefix='lstm_l%d_'%i))
- begin_state = stack.begin_state(func=mx.sym.Variable)
- _, states = stack.unroll(10, begin_state=begin_state, inputs=mx.sym.Variable('data'))
-
- state_names = [i.name for i in begin_state]
- mod = mx.mod.Module(mx.sym.Group(states), context=[mx.cpu(0), mx.cpu(1)],
- label_names=None, state_names=state_names)
- mod.bind(data_shapes=[('data', (5, 10))], label_shapes=None, for_training=False)
- mod.init_params()
- batch = mx.io.DataBatch(data=[mx.nd.zeros((5, 10))], label=[])
-
- mod.set_states(value=1)
- mod.forward(batch)
- out = mod.get_outputs(merge_multi_context=False)
- out1 = mod.get_outputs(merge_multi_context=True)
-
- mod.set_states(states=out)
- mod.forward(batch)
- out2 = mod.get_outputs(merge_multi_context=True)
-
- for x1, x2 in zip(out1, out2):
- assert not mx.test_utils.almost_equal(x1.asnumpy(), x2.asnumpy(), rtol=1e-3)
-
-
-@with_seed()
-def test_module_switch_bucket():
- vocab_dim = 5000
- num_hidden = 100
- num_embedding = 100
- num_layer = 2
- default_key = 10
- test_key = 5
- batch_size = 32
- contexts = [mx.cpu(0)]
- initializer = mx.init.Xavier(factor_type="in", magnitude=2.34)
-
- #generate symbols for an LSTM network
- def sym_gen(seq_len):
- data = mx.sym.Variable('data')
- label = mx.sym.Variable('softmax_label')
- embed = mx.sym.Embedding(data=data, input_dim=vocab_dim,
- output_dim=num_embedding)
- stack = mx.rnn.SequentialRNNCell()
- for i in range(num_layer):
- stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_'%i))
- outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
- pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
- pred = mx.sym.FullyConnected(data=pred, num_hidden=vocab_dim, name='pred')
-
- label = mx.sym.Reshape(label, shape=(-1,))
- pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-
- return pred, ('data',), ('softmax_label',)
-
- def create_bucketing_module(key):
- model = mx.mod.BucketingModule(
- sym_gen = sym_gen,
- default_bucket_key = key,
- context = contexts)
- model.bind([('data', (batch_size, key))],
- [('softmax_label', (batch_size, key))], True, False)
- model.init_params(initializer=initializer)
- return model
- #initialize the bucketing module with the default bucket key
- bucketing_model = create_bucketing_module(default_key)
- #check name
- assert bucketing_model.symbol.list_arguments()[1] == "embedding0_weight",\
- "Error in assigning names for args in BucketingModule"
-
- #switch to test_key
- bucketing_model.switch_bucket(test_key, [('data', (batch_size, test_key))],
- [('softmax_label', (batch_size, test_key))])
- total_bytes_before = bucketing_model._buckets[default_key]._total_exec_bytes
-
- #remove test_key and switch again
- del bucketing_model._buckets[test_key]
- bucketing_model.switch_bucket(test_key, [('data', (batch_size, test_key))],
- [('softmax_label', (batch_size, test_key))])
- total_bytes_after = bucketing_model._buckets[default_key]._total_exec_bytes
- #the default bucket is expected to reuse the bytes allocated
- assert total_bytes_after == total_bytes_before
-
# roywei: Getting rid of fixed seed as flakiness could not be reproduced,
# tracked at: https://github.com/apache/incubator-mxnet/issues/11705
@@ -510,134 +352,6 @@ def test_monitor():
break
assert(mon_result_counts == [2, 2, 1, 6, 6, 4])
-@with_seed()
-def test_executor_group():
- def get_rnn_sym(num_layers, num_words, num_hidden, num_embed, seq_len, sparse_embedding):
- stack = mx.rnn.SequentialRNNCell()
- for i in range(num_layers):
- stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i))
- data = mx.sym.Variable('data')
- label = mx.sym.Variable('softmax_label')
- if sparse_embedding:
- embed_weight = mx.sym.Variable('embed_weight', stype='row_sparse')
- embed = mx.sym.contrib.SparseEmbedding(data=data, input_dim=num_words,
- weight=embed_weight, output_dim=num_embed,
- name='embed')
- else:
- embed = mx.sym.Embedding(data=data, input_dim=num_words,
- output_dim=num_embed, name='embed')
-
- stack.reset()
- outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
- pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
- pred = mx.sym.FullyConnected(data=pred, num_hidden=num_words, name='pred')
-
- label = mx.sym.Reshape(label, shape=(-1,))
- pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
- return pred
-
- def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=None,
- extra_args=None, check_shared_grad=True):
- # Test shared data arrays
- for i in range(len(exec_grp_shared.execs)):
- # test same shared_data_arrays for two exec groups
- shared_data_array1 = exec_grp_shared.shared_data_arrays[i]
- shared_data_array2 = exec_grp_created.shared_data_arrays[i]
- if extra_args is not None:
- assert len(shared_data_array1) == len(extra_args),\
- "exec_grp_shared.shared_data_arrays[%d] should have same number of args as extra_args"
- assert len(shared_data_array1) == len(shared_data_array2),\
- "length of shared_data_array of the shared executor group not equal to the created executor group"
- for k, v in shared_data_array1.items():
- if extra_args is not None:
- assert k in extra_args, "arg %s is not in extra_args" % k
- assert k in shared_data_array2,\
- "arg %s of the shared executor group not in the shared_data_array of the created executor group" % k
- assert mx.test_utils.same_array(v, shared_data_array2[k])
-
- for data_name, array in exec_grp_shared.shared_data_arrays[i].items():
- assert data_name in exec_grp_created.shared_data_arrays[i], \
- "Shared input data '%s' is not in " \
- "shared_data_arrays of created executor group." % (data_name)
- assert mx.test_utils.same_array(array, exec_grp_created.shared_data_arrays[i][data_name]), \
- "Shared input data '%s' does not share memory." % (data_name)
-
- # Test shared argument arrays and gradient arrays
- exec_shared = exec_grp_shared.execs[i]
- exec_created = exec_grp_created.execs[i]
- if shared_arg_names is not None:
- # test shared arguments
- for arg_name in shared_arg_names:
- assert arg_name in exec_created.arg_dict, \
- "Shared argument '%s' is not in arg_dict of created executor group." % (arg_name)
- assert mx.test_utils.same_array(exec_shared.arg_dict[arg_name], exec_created.arg_dict[arg_name]), \
- "Shared argument '%s' does not share memory." % (arg_name)
- # test shared argument gradients
- if check_shared_grad:
- for arg_name in shared_arg_names:
- assert arg_name in exec_created.grad_dict, \
- "Shared argument gradient '%s' is not in " \
- "grad_dict of created executor group." % (arg_name)
- assert mx.test_utils.same_array(exec_shared.grad_dict[arg_name], \
- exec_created.grad_dict[arg_name]), \
- "Shared argument gradient '%s' does not share memory." % (arg_name)
-
- for arg_name, grad in exec_grp_shared.grad_req.items():
- assert grad == exec_grp_created.grad_req[arg_name], \
- "Gradient requirements for shared argument '%s' are inconsistent. " \
- "Shared executor group requires '%s' while created executor group requires '%s'" \
- %(arg_name, grad, exec_grp_created.grad_req[arg_name])
-
- def check_shared_exec_group(sparse_embedding):
- # generate an rnn sym with #layers=5
- sym = get_rnn_sym(num_layers=3, num_words=num_words, num_hidden=num_hidden,
- num_embed=num_embed, seq_len=max_bucket_size,
- sparse_embedding=sparse_embedding)
- arg_names1 = sym.list_arguments()
- input_names = [name[0] for name in data_shapes] + [name[0] for name in label_shapes]
- shared_arg_names = [name for name in arg_names1 if name not in input_names]
- exec_group1 = DataParallelExecutorGroup(symbol=sym, contexts=contexts,
- workload=workload, data_shapes=data_shapes,
- label_shapes=label_shapes, param_names=shared_arg_names,
- for_training=True, inputs_need_grad=False)
-
- # shared_data_arrays should only have input "data" and "softmax_label" arrays
- for i in range(len(contexts)):
- assert len(exec_group1.shared_data_arrays[i]) == len(input_names),\
- "exec_group1.shared_data_arrays[%d] should have the same number of names as in input_names" % i
- for name in input_names:
- assert name in exec_group1.shared_data_arrays[i],\
- "arg %s should be in exec_group1.shared_data_arrays[%d]" % (name, i)
-
- # generate an rnn sym with #layers=5
- sym = get_rnn_sym(num_layers=5, num_words=num_words, num_hidden=num_hidden,
- num_embed=num_embed, seq_len=max_bucket_size,
- sparse_embedding=sparse_embedding)
- arg_names2 = sym.list_arguments()
- exec_group2 = DataParallelExecutorGroup(symbol=sym, contexts=contexts,
- workload=workload, data_shapes=data_shapes,
- label_shapes=label_shapes, param_names=shared_arg_names,
- for_training=True, inputs_need_grad=False,
- shared_group=exec_group1)
- extra_args = [name for name in arg_names2 if name not in shared_arg_names]
- check_shared_grad = not sparse_embedding
- test_shared_exec_group(exec_grp_shared=exec_group1, exec_grp_created=exec_group2,
- shared_arg_names=shared_arg_names, extra_args=extra_args,
- check_shared_grad=check_shared_grad)
-
- contexts = [mx.cpu(0), mx.cpu(1)]
- workload = [1] * len(contexts)
- batch_size = 32
- max_bucket_size = 80
- num_words = 1000
- num_hidden = 100
- num_embed = 200
- data_shapes = [('data', (batch_size, max_bucket_size))]
- label_shapes = [('softmax_label', (batch_size, max_bucket_size))]
- sparse_embedding_opt = [True, False]
- for opt in sparse_embedding_opt:
- check_shared_exec_group(opt)
@with_seed()
def test_factorization_machine_module():
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index bdddf62..3a34f40 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -116,185 +116,6 @@ def test_rnn_with_new_param():
@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_lstm_sym():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='lstm', get_next_state=True, prefix='')
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.LSTMCell(H, prefix='l0_'))
- stack.add(mx.rnn.LSTMCell(H, prefix='l1_'))
- stack.add(mx.rnn.LSTMCell(H, prefix='l2_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_lstm_bidirectional():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='lstm',
- bidirectional=True, get_next_state=True, prefix='')
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.LSTMCell(H, prefix='l0_'),
- mx.rnn.LSTMCell(H, prefix='r0_'),
- output_prefix='bi_lstm_0_'))
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.LSTMCell(H, prefix='l1_'),
- mx.rnn.LSTMCell(H, prefix='r1_'),
- output_prefix='bi_lstm_1_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
- check_rnn_consistency(fused, stack, T, N, I, H, {'data': 'add', 'parameters': 'null'})
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_gru_sym():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='gru', get_next_state=True, prefix='')
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.GRUCell(H, prefix='l0_'))
- stack.add(mx.rnn.GRUCell(H, prefix='l1_'))
- stack.add(mx.rnn.GRUCell(H, prefix='l2_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_gru_bidirectional():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='gru',
- bidirectional=True, get_next_state=True, prefix='')
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.GRUCell(H, prefix='l0_'),
- mx.rnn.GRUCell(H, prefix='r0_'),
- output_prefix='bi_gru_0_'))
-
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.GRUCell(H, prefix='l1_'),
- mx.rnn.GRUCell(H, prefix='r1_'),
- output_prefix='bi_gru_1_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnntanh_sym():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='rnn_tanh', get_next_state=True, prefix='')
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l0_'))
- stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l1_'))
- stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l2_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnntanh_bidirectional():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='rnn_tanh',
- bidirectional=True, get_next_state=True, prefix='')
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.RNNCell(H, activation='tanh', prefix='l0_'),
- mx.rnn.RNNCell(H, activation='tanh', prefix='r0_'),
- output_prefix='bi_rnntanh_0_'))
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.RNNCell(H, activation='tanh', prefix='l1_'),
- mx.rnn.RNNCell(H, activation='tanh', prefix='r1_'),
- output_prefix='bi_rnntanh_1_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnnrelu_sym():
- T, N, I, H = 5, 32, 200, 200
-
- fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='rnn_relu', get_next_state=True, prefix='')
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l0_'))
- stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l1_'))
- stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l2_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write')
- check_rnn_consistency(fused, stack, T, N, I, H, 'add')
- check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnnrelu_bidirectional():
- Ts = [1, 5]
- Ns = [1, 32]
- Is = [32, 128, 512]
- Hs = [32, 128, 512]
- for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
- fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='rnn_relu',
- bidirectional=True, get_next_state=True, prefix='')
-
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.RNNCell(H, activation='relu', prefix='l0_'),
- mx.rnn.RNNCell(H, activation='relu', prefix='r0_'),
- output_prefix='bi_rnnrelu_0_'))
- stack.add(mx.rnn.BidirectionalCell(
- mx.rnn.RNNCell(H, activation='relu', prefix='l1_'),
- mx.rnn.RNNCell(H, activation='relu', prefix='r1_'),
- output_prefix='bi_rnnrelu_1_'))
-
- check_rnn_consistency(fused, stack, T, N, I, H, 'write', rtol=1e-2, atol=1e-2)
- check_rnn_consistency(fused, stack, T, N, I, H, 'add', rtol=1e-2, atol=1e-2)
- check_rnn_consistency(fused, stack, T, N, I, H, 'null', rtol=1e-2, atol=1e-2)
-
-@with_seed()
@pytest.mark.serial
def test_lstm_dropout():
X = mx.sym.Variable('x')
diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py
deleted file mode 100644
index ab333f9..0000000
--- a/tests/python/unittest/test_rnn.py
+++ /dev/null
@@ -1,311 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import mxnet as mx
-import numpy as np
-from numpy.testing import assert_allclose
-
-
-def test_deprecated():
- class RNNCell(mx.rnn.BaseRNNCell):
- """Simple recurrent neural network cell
-
- Parameters
- ----------
- num_hidden : int
- number of units in output symbol
- activation : str or Symbol, default 'tanh'
- type of activation function
- prefix : str, default 'rnn_'
- prefix for name of layers
- (and name of weight if params is None)
- params : RNNParams or None
- container for weight sharing between cells.
- created if None.
- """
- def __init__(self, num_hidden, activation='tanh', prefix='rnn_', params=None):
- super(RNNCell, self).__init__(prefix=prefix, params=params)
- self._num_hidden = num_hidden
- self._activation = activation
- self._iW = self.params.get('i2h_weight')
- self._iB = self.params.get('i2h_bias')
- self._hW = self.params.get('h2h_weight')
- self._hB = self.params.get('h2h_bias')
-
- @property
- def state_info(self):
- return [{'shape': (0, self._num_hidden), '__layout__': 'NC'}]
-
- @property
- def _gate_names(self):
- return ('',)
-
- def __call__(self, inputs, states):
- self._counter += 1
- name = '%st%d_'%(self._prefix, self._counter)
- i2h = mx.symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB,
- num_hidden=self._num_hidden,
- name='%si2h'%name)
- h2h = mx.symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB,
- num_hidden=self._num_hidden,
- name='%sh2h'%name)
- output = self._get_activation(i2h + h2h, self._activation,
- name='%sout'%name)
-
- return output, [output]
-
- cell = RNNCell(100, prefix='rnn_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_rnn():
- cell = mx.rnn.RNNCell(100, prefix='rnn_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_lstm():
- for activation_type in ['', 'relu', 'sigmoid', 'softrelu', 'tanh', 'softsign']:
- if activation_type == '':
- cell = mx.gluon.rnn.LSTMCell(100, prefix='rnn_')
- else:
- cell = mx.gluon.rnn.LSTMCell(100, prefix='rnn_', activation=activation_type, recurrent_activation=activation_type)
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.collect_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_lstm_forget_bias():
- forget_bias = 2.0
- stack = mx.rnn.SequentialRNNCell()
- stack.add(mx.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l0_'))
- stack.add(mx.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l1_'))
-
- dshape = (32, 1, 200)
- data = mx.sym.Variable('data')
-
- sym, _ = stack.unroll(1, data, merge_outputs=True)
- mod = mx.mod.Module(sym, label_names=None, context=mx.cpu(0))
- mod.bind(data_shapes=[('data', dshape)], label_shapes=None)
-
- mod.init_params()
-
- bias_argument = next(x for x in sym.list_arguments() if x.endswith('i2h_bias'))
- expected_bias = np.hstack([np.zeros((100,)),
- forget_bias * np.ones(100, ), np.zeros((2 * 100,))])
- assert_allclose(mod.get_params()[0][bias_argument].asnumpy(), expected_bias)
-
-
-def test_gru():
- cell = mx.rnn.GRUCell(100, prefix='rnn_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_residual():
- cell = mx.rnn.ResidualCell(mx.rnn.GRUCell(50, prefix='rnn_'))
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)]
- outputs, _ = cell.unroll(2, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == \
- ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- # assert outputs.list_outputs() == \
- # ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50))
- assert outs == [(10, 50), (10, 50)]
- outputs = outputs.eval(rnn_t0_data=mx.nd.ones((10, 50)),
- rnn_t1_data=mx.nd.ones((10, 50)),
- rnn_i2h_weight=mx.nd.zeros((150, 50)),
- rnn_i2h_bias=mx.nd.zeros((150,)),
- rnn_h2h_weight=mx.nd.zeros((150, 50)),
- rnn_h2h_bias=mx.nd.zeros((150,)))
- expected_outputs = np.ones((10, 50))
- assert np.array_equal(outputs[0].asnumpy(), expected_outputs)
- assert np.array_equal(outputs[1].asnumpy(), expected_outputs)
-
-
-def test_residual_bidirectional():
- cell = mx.rnn.ResidualCell(
- mx.rnn.BidirectionalCell(
- mx.rnn.GRUCell(25, prefix='rnn_l_'),
- mx.rnn.GRUCell(25, prefix='rnn_r_')))
-
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)]
- outputs, _ = cell.unroll(2, inputs, merge_outputs=False)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == \
- ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight',
- 'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight']
- # assert outputs.list_outputs() == \
- # ['bi_t0_plus_residual_output', 'bi_t1_plus_residual_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50))
- assert outs == [(10, 50), (10, 50)]
- outputs = outputs.eval(rnn_t0_data=mx.nd.ones((10, 50))+5,
- rnn_t1_data=mx.nd.ones((10, 50))+5,
- rnn_l_i2h_weight=mx.nd.zeros((75, 50)),
- rnn_l_i2h_bias=mx.nd.zeros((75,)),
- rnn_l_h2h_weight=mx.nd.zeros((75, 25)),
- rnn_l_h2h_bias=mx.nd.zeros((75,)),
- rnn_r_i2h_weight=mx.nd.zeros((75, 50)),
- rnn_r_i2h_bias=mx.nd.zeros((75,)),
- rnn_r_h2h_weight=mx.nd.zeros((75, 25)),
- rnn_r_h2h_bias=mx.nd.zeros((75,)))
- expected_outputs = np.ones((10, 50))+5
- assert np.array_equal(outputs[0].asnumpy(), expected_outputs)
- assert np.array_equal(outputs[1].asnumpy(), expected_outputs)
-
-
-def test_stack():
- cell = mx.rnn.SequentialRNNCell()
- for i in range(5):
- if i == 1:
- cell.add(mx.rnn.ResidualCell(mx.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i)))
- else:
- cell.add(mx.rnn.LSTMCell(100, prefix='rnn_stack%d_'%i))
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- keys = sorted(cell.params._params.keys())
- for i in range(5):
- assert 'rnn_stack%d_h2h_weight'%i in keys
- assert 'rnn_stack%d_h2h_bias'%i in keys
- assert 'rnn_stack%d_i2h_weight'%i in keys
- assert 'rnn_stack%d_i2h_bias'%i in keys
- assert outputs.list_outputs() == ['rnn_stack4_t0_out_output', 'rnn_stack4_t1_out_output', 'rnn_stack4_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_bidirectional():
- cell = mx.rnn.BidirectionalCell(
- mx.rnn.LSTMCell(100, prefix='rnn_l0_'),
- mx.rnn.LSTMCell(100, prefix='rnn_r0_'),
- output_prefix='rnn_bi_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert outputs.list_outputs() == ['rnn_bi_t0_output', 'rnn_bi_t1_output', 'rnn_bi_t2_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 200), (10, 200), (10, 200)]
-
-
-def test_zoneout():
- cell = mx.rnn.ZoneoutCell(mx.rnn.RNNCell(100, prefix='rnn_'), zoneout_outputs=0.5,
- zoneout_states=0.5)
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_unfuse():
- cell = mx.rnn.FusedRNNCell(100, num_layers=3, mode='lstm',
- prefix='test_', bidirectional=True,
- dropout=0.5)
- cell = cell.unfuse()
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert outputs.list_outputs() == ['test_bi_l2_t0_output', 'test_bi_l2_t1_output', 'test_bi_l2_t2_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
- assert outs == [(10, 200), (10, 200), (10, 200)]
-
-def test_convrnn():
- cell = mx.rnn.ConvRNNCell(input_shape = (1, 3, 16, 10), num_hidden=10,
- h2h_kernel=(3, 3), h2h_dilate=(1, 1),
- i2h_kernel=(3, 3), i2h_stride=(1, 1),
- i2h_pad=(1, 1), i2h_dilate=(1, 1),
- prefix='rnn_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10))
- assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)]
-
-def test_convlstm():
- cell = mx.rnn.ConvLSTMCell(input_shape = (1, 3, 16, 10), num_hidden=10,
- h2h_kernel=(3, 3), h2h_dilate=(1, 1),
- i2h_kernel=(3, 3), i2h_stride=(1, 1),
- i2h_pad=(1, 1), i2h_dilate=(1, 1),
- prefix='rnn_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10))
- assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)]
-
-def test_convgru():
- cell = mx.rnn.ConvGRUCell(input_shape = (1, 3, 16, 10), num_hidden=10,
- h2h_kernel=(3, 3), h2h_dilate=(1, 1),
- i2h_kernel=(3, 3), i2h_stride=(1, 1),
- i2h_pad=(1, 1), i2h_dilate=(1, 1),
- prefix='rnn_')
- inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
- outputs, _ = cell.unroll(3, inputs)
- outputs = mx.sym.Group(outputs)
- assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
- assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
- args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10))
- assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)]
-
-def test_encode_sentences():
- sentences = [['a','b','c'],['b','c','d']]
- dict = {'a':1, 'b':2, 'c':3}
- result, vocab = mx.rnn.io.encode_sentences(sentences, vocab=dict, invalid_label=-1, invalid_key='\n',
- start_label=0, unknown_token='UNK')
- print(result, vocab)
- assert vocab == {'a': 1, 'b': 2, 'c': 3, 'UNK': 0}
- assert result == [[1,2,3],[2,3,0]]
-