You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2020/06/08 17:40:51 UTC

[incubator-mxnet] branch master updated: remove mx.rnn APIs (#18507)

This is an automated email from the ASF dual-hosted git repository.

zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new 992ed3c  remove mx.rnn APIs (#18507)
992ed3c is described below

commit 992ed3c1ea449fdb1f4f7010dfd05d00ae88a020
Author: Haibin Lin <li...@gmail.com>
AuthorDate: Mon Jun 8 10:39:56 2020 -0700

    remove mx.rnn APIs (#18507)
    
    * remove mx.rnn APIs
    
    * fix test
    
    * update test
    
    Co-authored-by: Ubuntu <ub...@ip-172-31-37-108.ec2.internal>
    Co-authored-by: Lin <ha...@a483e7be4c92.ant.amazon.com>
---
 python/mxnet/__init__.py                           |    1 -
 python/mxnet/initializer.py                        |   54 -
 python/mxnet/rnn/__init__.py                       |   29 -
 python/mxnet/rnn/io.py                             |  210 ---
 python/mxnet/rnn/rnn.py                            |  121 --
 python/mxnet/rnn/rnn_cell.py                       | 1441 --------------------
 tests/python/gpu/test_contrib_amp.py               |   35 -
 tests/python/gpu/test_operator_gpu.py              |  108 --
 tests/python/quantization/test_quantization.py     |  163 ---
 tests/python/train/test_bucketing.py               |  125 --
 tests/python/unittest/test_contrib_control_flow.py |  193 ---
 tests/python/unittest/test_module.py               |  286 ----
 tests/python/unittest/test_operator.py             |  179 ---
 tests/python/unittest/test_rnn.py                  |  311 -----
 14 files changed, 3256 deletions(-)

diff --git a/python/mxnet/__init__.py b/python/mxnet/__init__.py
index 66b23d4..ae90e4b 100644
--- a/python/mxnet/__init__.py
+++ b/python/mxnet/__init__.py
@@ -84,7 +84,6 @@ from . import image as img
 
 from . import test_utils
 
-from . import rnn
 from . import gluon
 
 from . import _deferred_compute
diff --git a/python/mxnet/initializer.py b/python/mxnet/initializer.py
index efec644..51cb63a 100644
--- a/python/mxnet/initializer.py
+++ b/python/mxnet/initializer.py
@@ -713,57 +713,3 @@ class LSTMBias(Initializer):
         # gate of the 4 LSTM gates, we modify the according values.
         num_hidden = int(arr.shape[0] / 4)
         arr[num_hidden:2*num_hidden] = self.forget_bias
-
-
-@register
-class FusedRNN(Initializer):
-    """Initialize parameters for fused rnn layers.
-
-    Parameters
-    ----------
-    init : Initializer
-        initializer applied to unpacked weights. Fall back to global
-        initializer if None.
-    num_hidden : int
-        should be the same with arguments passed to FusedRNNCell.
-    num_layers : int
-        should be the same with arguments passed to FusedRNNCell.
-    mode : str
-        should be the same with arguments passed to FusedRNNCell.
-    bidirectional : bool
-        should be the same with arguments passed to FusedRNNCell.
-    forget_bias : float
-        should be the same with arguments passed to FusedRNNCell.
-    """
-    def __init__(self, init, num_hidden, num_layers, mode, bidirectional=False, forget_bias=1.0):
-        if isinstance(init, string_types):
-            klass, kwargs = json.loads(init)
-            init = registry._REGISTRY[klass.lower()](**kwargs)
-        super(FusedRNN, self).__init__(init=init.dumps() if init is not None else None,
-                                       num_hidden=num_hidden, num_layers=num_layers, mode=mode,
-                                       bidirectional=bidirectional, forget_bias=forget_bias)
-        self._init = init
-        self._num_hidden = num_hidden
-        self._num_layers = num_layers
-        self._mode = mode
-        self._bidirectional = bidirectional
-        self._forget_bias = forget_bias
-
-    def _init_weight(self, desc, arr): # pylint: disable=arguments-differ
-        from .rnn import rnn_cell
-        cell = rnn_cell.FusedRNNCell(self._num_hidden, self._num_layers,
-                                     self._mode, self._bidirectional,
-                                     forget_bias=self._forget_bias, prefix='')
-        args = cell.unpack_weights({'parameters': arr})
-        for name in args:
-            arg_desc = InitDesc(name, global_init=desc.global_init)
-            # for lstm bias, we use a custom initializer
-            # which adds a bias to the forget gate
-            if self._mode == 'lstm' and name.endswith("_f_bias"):
-                args[name][:] = self._forget_bias
-            elif self._init is None:
-                desc.global_init(arg_desc, args[name])
-            else:
-                self._init(arg_desc, args[name])
-
-        arr[:] = cell.pack_weights(args)['parameters']
diff --git a/python/mxnet/rnn/__init__.py b/python/mxnet/rnn/__init__.py
deleted file mode 100644
index dbf382e..0000000
--- a/python/mxnet/rnn/__init__.py
+++ /dev/null
@@ -1,29 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=wildcard-import
-"""Recurrent neural network module."""
-
-from . import rnn_cell
-from .rnn_cell import *
-
-from . import rnn
-from .rnn import *
-
-from . import io
-from .io import *
diff --git a/python/mxnet/rnn/io.py b/python/mxnet/rnn/io.py
deleted file mode 100644
index a10714b8..0000000
--- a/python/mxnet/rnn/io.py
+++ /dev/null
@@ -1,210 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=too-many-arguments, too-many-locals
-"""Definition of various recurrent neural network cells."""
-
-import bisect
-import random
-import numpy as np
-
-from ..io import DataIter, DataBatch, DataDesc
-from .. import ndarray
-
-def encode_sentences(sentences, vocab=None, invalid_label=-1, invalid_key='\n',
-                     start_label=0, unknown_token=None):
-    """Encode sentences and (optionally) build a mapping
-    from string tokens to integer indices. Unknown keys
-    will be added to vocabulary.
-
-    Parameters
-    ----------
-    sentences : list of list of str
-        A list of sentences to encode. Each sentence
-        should be a list of string tokens.
-    vocab : None or dict of str -> int
-        Optional input Vocabulary
-    invalid_label : int, default -1
-        Index for invalid token, like <end-of-sentence>
-    invalid_key : str, default '\\n'
-        Key for invalid token. Use '\\n' for end
-        of sentence by default.
-    start_label : int
-        lowest index.
-    unknown_token: str
-        Symbol to represent unknown token.
-        If not specified, unknown token will be skipped.
-
-    Returns
-    -------
-    result : list of list of int
-        encoded sentences
-    vocab : dict of str -> int
-        result vocabulary
-    """
-    idx = start_label
-    if vocab is None:
-        vocab = {invalid_key: invalid_label}
-        new_vocab = True
-    else:
-        new_vocab = False
-    res = []
-    for sent in sentences:
-        coded = []
-        for word in sent:
-            if word not in vocab:
-                assert (new_vocab or unknown_token), "Unknown token %s"%word
-                if idx == invalid_label:
-                    idx += 1
-                if unknown_token:
-                    word = unknown_token
-                vocab[word] = idx
-                idx += 1
-            coded.append(vocab[word])
-        res.append(coded)
-
-    return res, vocab
-
-class BucketSentenceIter(DataIter):
-    """Simple bucketing iterator for language model.
-    The label at each sequence step is the following token
-    in the sequence.
-
-    Parameters
-    ----------
-    sentences : list of list of int
-        Encoded sentences.
-    batch_size : int
-        Batch size of the data.
-    invalid_label : int, optional
-        Key for invalid label, e.g. <end-of-sentence>. The default is -1.
-    dtype : str, optional
-        Data type of the encoding. The default data type is 'float32'.
-    buckets : list of int, optional
-        Size of the data buckets. Automatically generated if None.
-    data_name : str, optional
-        Name of the data. The default name is 'data'.
-    label_name : str, optional
-        Name of the label. The default name is 'softmax_label'.
-    layout : str, optional
-        Format of data and label. 'NT' means (batch_size, length)
-        and 'TN' means (length, batch_size).
-    """
-    def __init__(self, sentences, batch_size, buckets=None, invalid_label=-1,
-                 data_name='data', label_name='softmax_label', dtype='float32',
-                 layout='NT'):
-        super(BucketSentenceIter, self).__init__()
-        if not buckets:
-            buckets = [i for i, j in enumerate(np.bincount([len(s) for s in sentences]))
-                       if j >= batch_size]
-        buckets.sort()
-
-        ndiscard = 0
-        self.data = [[] for _ in buckets]
-        valid_buckets = {}
-        for item in range(len(buckets)):
-            valid_buckets[item] = 0
-
-        for i, sent in enumerate(sentences):
-            buck = bisect.bisect_left(buckets, len(sent))
-            valid_buckets[buck] = 1
-            if buck == len(buckets):
-                ndiscard += 1
-                continue
-            buff = np.full((buckets[buck],), invalid_label, dtype=dtype)
-            buff[:len(sent)] = sent
-            self.data[buck].append(buff)
-        buckets = [j for i, j in enumerate(buckets) if valid_buckets[i] == 1]
-        self.data = [np.asarray(i, dtype=dtype) for i in self.data if i]
-
-        print("WARNING: discarded %d sentences longer than the largest bucket."%ndiscard)
-
-        self.batch_size = batch_size
-        self.buckets = buckets
-        self.data_name = data_name
-        self.label_name = label_name
-        self.dtype = dtype
-        self.invalid_label = invalid_label
-        self.nddata = []
-        self.ndlabel = []
-        self.major_axis = layout.find('N')
-        self.layout = layout
-        self.default_bucket_key = max(buckets)
-
-        if self.major_axis == 0:
-            self.provide_data = [DataDesc(
-                name=self.data_name, shape=(batch_size, self.default_bucket_key),
-                layout=self.layout)]
-            self.provide_label = [DataDesc(
-                name=self.label_name, shape=(batch_size, self.default_bucket_key),
-                layout=self.layout)]
-        elif self.major_axis == 1:
-            self.provide_data = [DataDesc(
-                name=self.data_name, shape=(self.default_bucket_key, batch_size),
-                layout=self.layout)]
-            self.provide_label = [DataDesc(
-                name=self.label_name, shape=(self.default_bucket_key, batch_size),
-                layout=self.layout)]
-        else:
-            raise ValueError("Invalid layout %s: Must by NT (batch major) or TN (time major)")
-
-        self.idx = []
-        for i, buck in enumerate(self.data):
-            self.idx.extend([(i, j) for j in range(0, len(buck) - batch_size + 1, batch_size)])
-        self.curr_idx = 0
-
-        self.reset()
-
-    def reset(self):
-        """Resets the iterator to the beginning of the data."""
-        self.curr_idx = 0
-        random.shuffle(self.idx)
-        for buck in self.data:
-            np.random.shuffle(buck)
-
-        self.nddata = []
-        self.ndlabel = []
-        for buck in self.data:
-            label = np.empty_like(buck)
-            label[:, :-1] = buck[:, 1:]
-            label[:, -1] = self.invalid_label
-            self.nddata.append(ndarray.array(buck, dtype=self.dtype))
-            self.ndlabel.append(ndarray.array(label, dtype=self.dtype))
-
-    def next(self):
-        """Returns the next batch of data."""
-        if self.curr_idx == len(self.idx):
-            raise StopIteration
-        i, j = self.idx[self.curr_idx]
-        self.curr_idx += 1
-
-        if self.major_axis == 1:
-            data = self.nddata[i][j:j+self.batch_size].T
-            label = self.ndlabel[i][j:j+self.batch_size].T
-        else:
-            data = self.nddata[i][j:j+self.batch_size]
-            label = self.ndlabel[i][j:j+self.batch_size]
-
-        return DataBatch([data], [label], pad=0,
-                         bucket_key=self.buckets[i],
-                         provide_data=[DataDesc(
-                             name=self.data_name, shape=data.shape,
-                             layout=self.layout)],
-                         provide_label=[DataDesc(
-                             name=self.label_name, shape=label.shape,
-                             layout=self.layout)])
diff --git a/python/mxnet/rnn/rnn.py b/python/mxnet/rnn/rnn.py
deleted file mode 100644
index 0255c55..0000000
--- a/python/mxnet/rnn/rnn.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=too-many-arguments, no-member
-"""Functions for constructing recurrent neural networks."""
-import warnings
-
-from ..model import save_checkpoint, load_checkpoint
-from .rnn_cell import BaseRNNCell
-
-def rnn_unroll(cell, length, inputs=None, begin_state=None, input_prefix='', layout='NTC'):
-    """Deprecated. Please use cell.unroll instead"""
-    warnings.warn('rnn_unroll is deprecated. Please call cell.unroll directly.')
-    return cell.unroll(length=length, inputs=inputs, begin_state=begin_state,
-                       input_prefix=input_prefix, layout=layout)
-
-def save_rnn_checkpoint(cells, prefix, epoch, symbol, arg_params, aux_params):
-    """Save checkpoint for model using RNN cells.
-    Unpacks weight before saving.
-
-    Parameters
-    ----------
-    cells : mxnet.rnn.RNNCell or list of RNNCells
-        The RNN cells used by this symbol.
-    prefix : str
-        Prefix of model name.
-    epoch : int
-        The epoch number of the model.
-    symbol : Symbol
-        The input symbol
-    arg_params : dict of str to NDArray
-        Model parameter, dict of name to NDArray of net's weights.
-    aux_params : dict of str to NDArray
-        Model parameter, dict of name to NDArray of net's auxiliary states.
-
-    Notes
-    -----
-    - ``prefix-symbol.json`` will be saved for symbol.
-    - ``prefix-epoch.params`` will be saved for parameters.
-    """
-    if isinstance(cells, BaseRNNCell):
-        cells = [cells]
-    for cell in cells:
-        arg_params = cell.unpack_weights(arg_params)
-    save_checkpoint(prefix, epoch, symbol, arg_params, aux_params)
-
-def load_rnn_checkpoint(cells, prefix, epoch):
-    """Load model checkpoint from file.
-    Pack weights after loading.
-
-    Parameters
-    ----------
-    cells : mxnet.rnn.RNNCell or list of RNNCells
-        The RNN cells used by this symbol.
-    prefix : str
-        Prefix of model name.
-    epoch : int
-        Epoch number of model we would like to load.
-
-    Returns
-    -------
-    symbol : Symbol
-        The symbol configuration of computation network.
-    arg_params : dict of str to NDArray
-        Model parameter, dict of name to NDArray of net's weights.
-    aux_params : dict of str to NDArray
-        Model parameter, dict of name to NDArray of net's auxiliary states.
-
-    Notes
-    -----
-    - symbol will be loaded from ``prefix-symbol.json``.
-    - parameters will be loaded from ``prefix-epoch.params``.
-    """
-    sym, arg, aux = load_checkpoint(prefix, epoch)
-    if isinstance(cells, BaseRNNCell):
-        cells = [cells]
-    for cell in cells:
-        arg = cell.pack_weights(arg)
-
-    return sym, arg, aux
-
-def do_rnn_checkpoint(cells, prefix, period=1):
-    """Make a callback to checkpoint Module to prefix every epoch.
-    unpacks weights used by cells before saving.
-
-    Parameters
-    ----------
-    cells : mxnet.rnn.RNNCell or list of RNNCells
-        The RNN cells used by this symbol.
-    prefix : str
-        The file prefix to checkpoint to
-    period : int
-        How many epochs to wait before checkpointing. Default is 1.
-
-    Returns
-    -------
-    callback : function
-        The callback function that can be passed as iter_end_callback to fit.
-    """
-    period = int(max(1, period))
-    # pylint: disable=unused-argument
-    def _callback(iter_no, sym=None, arg=None, aux=None):
-        """The checkpoint function."""
-        if (iter_no + 1) % period == 0:
-            save_rnn_checkpoint(cells, prefix, iter_no+1, sym, arg, aux)
-    return _callback
diff --git a/python/mxnet/rnn/rnn_cell.py b/python/mxnet/rnn/rnn_cell.py
deleted file mode 100644
index f6f99e8..0000000
--- a/python/mxnet/rnn/rnn_cell.py
+++ /dev/null
@@ -1,1441 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# coding: utf-8
-# pylint: disable=no-member, invalid-name, protected-access, no-self-use
-# pylint: disable=too-many-branches, too-many-arguments, no-self-use
-# pylint: disable=too-many-lines
-"""Definition of various recurrent neural network cells."""
-
-import warnings
-import functools
-
-from .. import symbol, init, ndarray
-from ..base import string_types, numeric_types
-
-
-def _cells_state_shape(cells):
-    return sum([c.state_shape for c in cells], [])
-
-def _cells_state_info(cells):
-    return sum([c.state_info for c in cells], [])
-
-def _cells_begin_state(cells, **kwargs):
-    return sum([c.begin_state(**kwargs) for c in cells], [])
-
-def _cells_unpack_weights(cells, args):
-    for cell in cells:
-        args = cell.unpack_weights(args)
-    return args
-
-def _cells_pack_weights(cells, args):
-    for cell in cells:
-        args = cell.pack_weights(args)
-    return args
-
-def _normalize_sequence(length, inputs, layout, merge, in_layout=None):
-    assert inputs is not None, \
-        "unroll(inputs=None) has been deprecated. " \
-        "Please create input variables outside unroll."
-
-    axis = layout.find('T')
-    in_axis = in_layout.find('T') if in_layout is not None else axis
-    if isinstance(inputs, symbol.Symbol):
-        if merge is False:
-            assert len(inputs.list_outputs()) == 1, \
-                "unroll doesn't allow grouped symbol as input. Please convert " \
-                "to list with list(inputs) first or let unroll handle splitting."
-            inputs = list(symbol.split(inputs, axis=in_axis, num_outputs=length,
-                                       squeeze_axis=1))
-    else:
-        assert length is None or len(inputs) == length
-        if merge is True:
-            inputs = [symbol.expand_dims(i, axis=axis) for i in inputs]
-            inputs = symbol.Concat(*inputs, dim=axis)
-            in_axis = axis
-
-    if isinstance(inputs, symbol.Symbol) and axis != in_axis:
-        inputs = symbol.swapaxes(inputs, dim0=axis, dim1=in_axis)
-
-    return inputs, axis
-
-
-class RNNParams(object):
-    """Container for holding variables.
-    Used by RNN cells for parameter sharing between cells.
-
-    Parameters
-    ----------
-    prefix : str
-        Names of all variables created by this container will
-        be prepended with prefix.
-    """
-    def __init__(self, prefix=''):
-        self._prefix = prefix
-        self._params = {}
-
-    def get(self, name, **kwargs):
-        """Get the variable given a name if one exists or create a new one if missing.
-
-        Parameters
-        ----------
-        name : str
-            name of the variable
-        **kwargs :
-            more arguments that's passed to symbol.Variable
-        """
-        name = self._prefix + name
-        if name not in self._params:
-            self._params[name] = symbol.Variable(name, **kwargs)
-        return self._params[name]
-
-
-class BaseRNNCell(object):
-    """Abstract base class for RNN cells
-
-    Parameters
-    ----------
-    prefix : str, optional
-        Prefix for names of layers
-        (this prefix is also used for names of weights if `params` is None
-        i.e. if `params` are being created and not reused)
-    params : RNNParams, default None.
-        Container for weight sharing between cells.
-        A new RNNParams container is created if `params` is None.
-    """
-    def __init__(self, prefix='', params=None):
-        if params is None:
-            params = RNNParams(prefix)
-            self._own_params = True
-        else:
-            self._own_params = False
-        self._prefix = prefix
-        self._params = params
-        self._modified = False
-
-        self.reset()
-
-    def reset(self):
-        """Reset before re-using the cell for another graph."""
-        self._init_counter = -1
-        self._counter = -1
-        if hasattr(self, '_cells'):
-            for cell in self._cells:
-                cell.reset()
-
-    def __call__(self, inputs, states):
-        """Unroll the RNN for one time step.
-
-        Parameters
-        ----------
-        inputs : sym.Variable
-            input symbol, 2D, batch * num_units
-        states : list of sym.Variable
-            RNN state from previous step or the output of begin_state().
-
-        Returns
-        -------
-        output : Symbol
-            Symbol corresponding to the output from the RNN when unrolling
-            for a single time step.
-        states : nested list of Symbol
-            The new state of this RNN after this unrolling.
-            The type of this symbol is same as the output of begin_state().
-            This can be used as input state to the next time step
-            of this RNN.
-
-        See Also
-        --------
-        begin_state: This function can provide the states for the first time step.
-        unroll: This function unrolls an RNN for a given number of (>=1) time steps.
-        """
-        raise NotImplementedError()
-
-    @property
-    def params(self):
-        """Parameters of this cell"""
-        self._own_params = False
-        return self._params
-
-    @property
-    def state_info(self):
-        """shape and layout information of states"""
-        raise NotImplementedError()
-
-    @property
-    def state_shape(self):
-        """shape(s) of states"""
-        return [ele['shape'] for ele in self.state_info]
-
-    @property
-    def _gate_names(self):
-        """name(s) of gates"""
-        return ()
-
-    def begin_state(self, func=symbol.zeros, **kwargs):
-        """Initial state for this cell.
-
-        Parameters
-        ----------
-        func : callable, default symbol.zeros
-            Function for creating initial state. Can be symbol.zeros,
-            symbol.uniform, symbol.Variable etc.
-            Use symbol.Variable if you want to directly
-            feed input as states.
-        **kwargs :
-            more keyword arguments passed to func. For example
-            mean, std, dtype, etc.
-
-        Returns
-        -------
-        states : nested list of Symbol
-            Starting states for the first RNN step.
-        """
-        assert not self._modified, \
-            "After applying modifier cells (e.g. DropoutCell) the base " \
-            "cell cannot be called directly. Call the modifier cell instead."
-        states = []
-        for info in self.state_info:
-            self._init_counter += 1
-            if info is None:
-                state = func(name='%sbegin_state_%d'%(self._prefix, self._init_counter),
-                             **kwargs)
-            else:
-                kwargs.update(info)
-                state = func(name='%sbegin_state_%d'%(self._prefix, self._init_counter),
-                             **kwargs)
-            states.append(state)
-        return states
-
-    def unpack_weights(self, args):
-        """Unpack fused weight matrices into separate
-        weight matrices.
-
-        For example, say you use a module object `mod` to run a network that has an lstm cell.
-        In `mod.get_params()[0]`, the lstm parameters are all represented as a single big vector.
-        `cell.unpack_weights(mod.get_params()[0])` will unpack this vector into a dictionary of
-        more readable lstm parameters - c, f, i, o gates for i2h (input to hidden) and
-        h2h (hidden to hidden) weights.
-
-        Parameters
-        ----------
-        args : dict of str -> NDArray
-            Dictionary containing packed weights.
-            usually from `Module.get_params()[0]`.
-
-        Returns
-        -------
-        args : dict of str -> NDArray
-            Dictionary with unpacked weights associated with
-            this cell.
-
-        See Also
-        --------
-        pack_weights: Performs the reverse operation of this function.
-        """
-        args = args.copy()
-        if not self._gate_names:
-            return args
-        h = self._num_hidden
-        for group_name in ['i2h', 'h2h']:
-            weight = args.pop('%s%s_weight'%(self._prefix, group_name))
-            bias = args.pop('%s%s_bias' % (self._prefix, group_name))
-            for j, gate in enumerate(self._gate_names):
-                wname = '%s%s%s_weight' % (self._prefix, group_name, gate)
-                args[wname] = weight[j*h:(j+1)*h].copy()
-                bname = '%s%s%s_bias' % (self._prefix, group_name, gate)
-                args[bname] = bias[j*h:(j+1)*h].copy()
-        return args
-
-    def pack_weights(self, args):
-        """Pack separate weight matrices into a single packed
-        weight.
-
-        Parameters
-        ----------
-        args : dict of str -> NDArray
-            Dictionary containing unpacked weights.
-
-        Returns
-        -------
-        args : dict of str -> NDArray
-            Dictionary with packed weights associated with
-            this cell.
-        """
-        args = args.copy()
-        if not self._gate_names:
-            return args
-        for group_name in ['i2h', 'h2h']:
-            weight = []
-            bias = []
-            for gate in self._gate_names:
-                wname = '%s%s%s_weight'%(self._prefix, group_name, gate)
-                weight.append(args.pop(wname))
-                bname = '%s%s%s_bias'%(self._prefix, group_name, gate)
-                bias.append(args.pop(bname))
-            args['%s%s_weight'%(self._prefix, group_name)] = ndarray.concatenate(weight)
-            args['%s%s_bias'%(self._prefix, group_name)] = ndarray.concatenate(bias)
-        return args
-
-    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
-        """Unroll an RNN cell across time steps.
-
-        Parameters
-        ----------
-        length : int
-            Number of steps to unroll.
-        inputs : Symbol, list of Symbol, or None
-            If `inputs` is a single Symbol (usually the output
-            of Embedding symbol), it should have shape
-            (batch_size, length, ...) if layout == 'NTC',
-            or (length, batch_size, ...) if layout == 'TNC'.
-
-            If `inputs` is a list of symbols (usually output of
-            previous unroll), they should all have shape
-            (batch_size, ...).
-        begin_state : nested list of Symbol, default None
-            Input states created by `begin_state()`
-            or output state of another cell.
-            Created from `begin_state()` if None.
-        layout : str, optional
-            `layout` of input symbol. Only used if inputs
-            is a single Symbol.
-        merge_outputs : bool, optional
-            If False, return outputs as a list of Symbols.
-            If True, concatenate output across time steps
-            and return a single symbol with shape
-            (batch_size, length, ...) if layout == 'NTC',
-            or (length, batch_size, ...) if layout == 'TNC'.
-            If None, output whatever is faster.
-
-        Returns
-        -------
-        outputs : list of Symbol or Symbol
-            Symbol (if `merge_outputs` is True) or list of Symbols
-            (if `merge_outputs` is False) corresponding to the output from
-            the RNN from this unrolling.
-
-        states : nested list of Symbol
-            The new state of this RNN after this unrolling.
-            The type of this symbol is same as the output of begin_state().
-        """
-        self.reset()
-
-        inputs, _ = _normalize_sequence(length, inputs, layout, False)
-        if begin_state is None:
-            begin_state = self.begin_state()
-
-        states = begin_state
-        outputs = []
-        for i in range(length):
-            output, states = self(inputs[i], states)
-            outputs.append(output)
-
-        outputs, _ = _normalize_sequence(length, outputs, layout, merge_outputs)
-
-        return outputs, states
-
-    #pylint: disable=no-self-use
-    def _get_activation(self, inputs, activation, **kwargs):
-        """Get activation function. Convert if is string"""
-        if isinstance(activation, string_types):
-            return symbol.Activation(inputs, act_type=activation, **kwargs)
-        else:
-            return activation(inputs, **kwargs)
-
-
-class RNNCell(BaseRNNCell):
-    """Simple recurrent neural network cell.
-
-    Parameters
-    ----------
-    num_hidden : int
-        Number of units in output symbol.
-    activation : str or Symbol, default 'tanh'
-        Type of activation function. Options are 'relu' and 'tanh'.
-    prefix : str, default ``'rnn_'``
-        Prefix for name of layers (and name of weight if params is None).
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    """
-    def __init__(self, num_hidden, activation='tanh', prefix='rnn_', params=None):
-        super(RNNCell, self).__init__(prefix=prefix, params=params)
-        self._num_hidden = num_hidden
-        self._activation = activation
-        self._iW = self.params.get('i2h_weight')
-        self._iB = self.params.get('i2h_bias')
-        self._hW = self.params.get('h2h_weight')
-        self._hB = self.params.get('h2h_bias')
-
-    @property
-    def state_info(self):
-        return [{'shape': (0, self._num_hidden), '__layout__': 'NC'}]
-
-    @property
-    def _gate_names(self):
-        return ('',)
-
-    def __call__(self, inputs, states):
-        self._counter += 1
-        name = '%st%d_'%(self._prefix, self._counter)
-        i2h = symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB,
-                                    num_hidden=self._num_hidden,
-                                    name='%si2h'%name)
-        h2h = symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB,
-                                    num_hidden=self._num_hidden,
-                                    name='%sh2h'%name)
-        output = self._get_activation(i2h + h2h, self._activation,
-                                      name='%sout'%name)
-
-        return output, [output]
-
-
-class LSTMCell(BaseRNNCell):
-    """Long-Short Term Memory (LSTM) network cell.
-
-    Parameters
-    ----------
-    num_hidden : int
-        Number of units in output symbol.
-    prefix : str, default ``'lstm_'``
-        Prefix for name of layers (and name of weight if params is None).
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    forget_bias : bias added to forget gate, default 1.0.
-        Jozefowicz et al. 2015 recommends setting this to 1.0
-    """
-    def __init__(self, num_hidden, prefix='lstm_', params=None, forget_bias=1.0):
-        super(LSTMCell, self).__init__(prefix=prefix, params=params)
-
-        self._num_hidden = num_hidden
-        self._iW = self.params.get('i2h_weight')
-        self._hW = self.params.get('h2h_weight')
-        # we add the forget_bias to i2h_bias, this adds the bias to the forget gate activation
-        self._iB = self.params.get('i2h_bias', init=init.LSTMBias(forget_bias=forget_bias))
-        self._hB = self.params.get('h2h_bias')
-
-    @property
-    def state_info(self):
-        return [{'shape': (0, self._num_hidden), '__layout__': 'NC'},
-                {'shape': (0, self._num_hidden), '__layout__': 'NC'}]
-
-    @property
-    def _gate_names(self):
-        return ['_i', '_f', '_c', '_o']
-
-    def __call__(self, inputs, states):
-        self._counter += 1
-        name = '%st%d_'%(self._prefix, self._counter)
-        i2h = symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB,
-                                    num_hidden=self._num_hidden*4,
-                                    name='%si2h'%name)
-        h2h = symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB,
-                                    num_hidden=self._num_hidden*4,
-                                    name='%sh2h'%name)
-        gates = i2h + h2h
-        slice_gates = symbol.SliceChannel(gates, num_outputs=4,
-                                          name="%sslice"%name)
-        in_gate = symbol.Activation(slice_gates[0], act_type="sigmoid",
-                                    name='%si'%name)
-        forget_gate = symbol.Activation(slice_gates[1], act_type="sigmoid",
-                                        name='%sf'%name)
-        in_transform = symbol.Activation(slice_gates[2], act_type="tanh",
-                                         name='%sc'%name)
-        out_gate = symbol.Activation(slice_gates[3], act_type="sigmoid",
-                                     name='%so'%name)
-        next_c = symbol._internal._plus(forget_gate * states[1], in_gate * in_transform,
-                                        name='%sstate'%name)
-        next_c._set_attr(force_mirroring='0')
-        # Cell states are excluded from being mirrored. The reason is because
-        # they do not pass through the fully-connected layers and will
-        # significantly increase the overall mirroring depth, incurring large
-        # performance overhead.
-        next_h = symbol._internal._mul(out_gate, symbol.Activation(next_c, act_type="tanh"),
-                                       name='%sout'%name)
-
-        return next_h, [next_h, next_c]
-
-
-class GRUCell(BaseRNNCell):
-    """Gated Rectified Unit (GRU) network cell.
-    Note: this is an implementation of the cuDNN version of GRUs
-    (slight modification compared to Cho et al. 2014).
-
-    Parameters
-    ----------
-    num_hidden : int
-        Number of units in output symbol.
-    prefix : str, default ``'gru_'``
-        Prefix for name of layers (and name of weight if params is None).
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    """
-    def __init__(self, num_hidden, prefix='gru_', params=None):
-        super(GRUCell, self).__init__(prefix=prefix, params=params)
-        self._num_hidden = num_hidden
-        self._iW = self.params.get("i2h_weight")
-        self._iB = self.params.get("i2h_bias")
-        self._hW = self.params.get("h2h_weight")
-        self._hB = self.params.get("h2h_bias")
-
-    @property
-    def state_info(self):
-        return [{'shape': (0, self._num_hidden),
-                 '__layout__': 'NC'}]
-
-    @property
-    def _gate_names(self):
-        return ['_r', '_z', '_o']
-
-    def __call__(self, inputs, states):
-        # pylint: disable=too-many-locals
-        self._counter += 1
-
-        seq_idx = self._counter
-        name = '%st%d_' % (self._prefix, seq_idx)
-        prev_state_h = states[0]
-
-        i2h = symbol.FullyConnected(data=inputs,
-                                    weight=self._iW,
-                                    bias=self._iB,
-                                    num_hidden=self._num_hidden * 3,
-                                    name="%s_i2h" % name)
-        h2h = symbol.FullyConnected(data=prev_state_h,
-                                    weight=self._hW,
-                                    bias=self._hB,
-                                    num_hidden=self._num_hidden * 3,
-                                    name="%s_h2h" % name)
-        # pylint: disable=unbalanced-tuple-unpacking
-        i2h_r, i2h_z, i2h = symbol.SliceChannel(i2h, num_outputs=3, name="%s_i2h_slice" % name)
-        h2h_r, h2h_z, h2h = symbol.SliceChannel(h2h, num_outputs=3, name="%s_h2h_slice" % name)
-
-        reset_gate = symbol.Activation(i2h_r + h2h_r, act_type="sigmoid",
-                                       name="%s_r_act" % name)
-        update_gate = symbol.Activation(i2h_z + h2h_z, act_type="sigmoid",
-                                        name="%s_z_act" % name)
-
-        next_h_tmp = symbol.Activation(i2h + reset_gate * h2h, act_type="tanh",
-                                       name="%s_h_act" % name)
-
-        next_h = symbol._internal._plus((1. - update_gate) * next_h_tmp, update_gate * prev_state_h,
-                                        name='%sout' % name)
-
-        return next_h, [next_h]
-
-
-class FusedRNNCell(BaseRNNCell):
-    """Fusing RNN layers across time step into one kernel.
-    Improves speed but is less flexible. Currently only
-    supported if using cuDNN on GPU.
-
-    Parameters
-    ----------
-    num_hidden : int
-        Number of units in output symbol.
-    num_layers : int, default 1
-        Number of layers in the cell.
-    mode : str, default 'lstm'
-        Type of RNN. options are 'rnn_relu', 'rnn_tanh', 'lstm', 'gru'.
-    bidirectional : bool, default False
-        Whether to use bidirectional unroll. The output dimension size is doubled if bidrectional.
-    dropout : float, default 0.
-        Fraction of the input that gets dropped out during training time.
-    get_next_state : bool, default False
-        Whether to return the states that can be used as starting states next time.
-    forget_bias : bias added to forget gate, default 1.0.
-        Jozefowicz et al. 2015 recommends setting this to 1.0
-    prefix : str, default ``'$mode_'`` such as ``'lstm_'``
-        Prefix for names of layers
-        (this prefix is also used for names of weights if `params` is None
-        i.e. if `params` are being created and not reused)
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    """
-    def __init__(self, num_hidden, num_layers=1, mode='lstm', bidirectional=False,
-                 dropout=0., get_next_state=False, forget_bias=1.0,
-                 prefix=None, params=None):
-        if prefix is None:
-            prefix = '%s_'%mode
-        super(FusedRNNCell, self).__init__(prefix=prefix, params=params)
-        self._num_hidden = num_hidden
-        self._num_layers = num_layers
-        self._mode = mode
-        self._bidirectional = bidirectional
-        self._dropout = dropout
-        self._get_next_state = get_next_state
-        self._directions = ['l', 'r'] if bidirectional else ['l']
-
-        initializer = init.FusedRNN(None, num_hidden, num_layers, mode,
-                                    bidirectional, forget_bias)
-        self._parameter = self.params.get('parameters', init=initializer)
-
-    @property
-    def state_info(self):
-        b = self._bidirectional + 1
-        n = (self._mode == 'lstm') + 1
-        return [{'shape': (b*self._num_layers, 0, self._num_hidden), '__layout__': 'LNC'}
-                for _ in range(n)]
-
-    @property
-    def _gate_names(self):
-        return {'rnn_relu': [''],
-                'rnn_tanh': [''],
-                'lstm': ['_i', '_f', '_c', '_o'],
-                'gru': ['_r', '_z', '_o']}[self._mode]
-
-    @property
-    def _num_gates(self):
-        return len(self._gate_names)
-
-    def _slice_weights(self, arr, li, lh):
-        """slice fused rnn weights"""
-        args = {}
-        gate_names = self._gate_names
-        directions = self._directions
-
-        b = len(directions)
-        p = 0
-        for layer in range(self._num_layers):
-            for direction in directions:
-                for gate in gate_names:
-                    name = '%s%s%d_i2h%s_weight'%(self._prefix, direction, layer, gate)
-                    if layer > 0:
-                        size = b*lh*lh
-                        args[name] = arr[p:p+size].reshape((lh, b*lh))
-                    else:
-                        size = li*lh
-                        args[name] = arr[p:p+size].reshape((lh, li))
-                    p += size
-                for gate in gate_names:
-                    name = '%s%s%d_h2h%s_weight'%(self._prefix, direction, layer, gate)
-                    size = lh**2
-                    args[name] = arr[p:p+size].reshape((lh, lh))
-                    p += size
-
-        for layer in range(self._num_layers):
-            for direction in directions:
-                for gate in gate_names:
-                    name = '%s%s%d_i2h%s_bias'%(self._prefix, direction, layer, gate)
-                    args[name] = arr[p:p+lh]
-                    p += lh
-                for gate in gate_names:
-                    name = '%s%s%d_h2h%s_bias'%(self._prefix, direction, layer, gate)
-                    args[name] = arr[p:p+lh]
-                    p += lh
-
-        assert p == arr.size, "Invalid parameters size for FusedRNNCell"
-        return args
-
-    def unpack_weights(self, args):
-        args = args.copy()
-        arr = args.pop(self._parameter.name)
-        b = len(self._directions)
-        m = self._num_gates
-        h = self._num_hidden
-        num_input = arr.size//b//h//m - (self._num_layers - 1)*(h+b*h+2) - h - 2
-
-        nargs = self._slice_weights(arr, num_input, self._num_hidden)
-        args.update({name: nd.copy() for name, nd in nargs.items()})
-        return args
-
-    def pack_weights(self, args):
-        args = args.copy()
-        b = self._bidirectional + 1
-        m = self._num_gates
-        c = self._gate_names
-        h = self._num_hidden
-        w0 = args['%sl0_i2h%s_weight'%(self._prefix, c[0])]
-        num_input = w0.shape[1]
-        total = (num_input+h+2)*h*m*b + (self._num_layers-1)*m*h*(h+b*h+2)*b
-
-        arr = ndarray.zeros((total,), ctx=w0.context, dtype=w0.dtype)
-        for name, nd in self._slice_weights(arr, num_input, h).items():
-            nd[:] = args.pop(name)
-        args[self._parameter.name] = arr
-        return args
-
-    def __call__(self, inputs, states):
-        raise NotImplementedError("FusedRNNCell cannot be stepped. Please use unroll")
-
-    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
-        self.reset()
-
-        inputs, axis = _normalize_sequence(length, inputs, layout, True)
-        if axis == 1:
-            warnings.warn("NTC layout detected. Consider using "
-                          "TNC for FusedRNNCell for faster speed")
-            inputs = symbol.swapaxes(inputs, dim1=0, dim2=1)
-        else:
-            assert axis == 0, "Unsupported layout %s"%layout
-        if begin_state is None:
-            begin_state = self.begin_state()
-
-        states = begin_state
-        if self._mode == 'lstm':
-            states = {'state': states[0], 'state_cell': states[1]} # pylint: disable=redefined-variable-type
-        else:
-            states = {'state': states[0]}
-
-        rnn = symbol.RNN(data=inputs, parameters=self._parameter,
-                         state_size=self._num_hidden, num_layers=self._num_layers,
-                         bidirectional=self._bidirectional, p=self._dropout,
-                         state_outputs=self._get_next_state,
-                         mode=self._mode, name=self._prefix+'rnn',
-                         **states)
-
-        attr = {'__layout__' : 'LNC'}
-        if not self._get_next_state:
-            outputs, states = rnn, []
-        elif self._mode == 'lstm':
-            rnn[1]._set_attr(**attr)
-            rnn[2]._set_attr(**attr)
-            outputs, states = rnn[0], [rnn[1], rnn[2]]
-        else:
-            rnn[1]._set_attr(**attr)
-            outputs, states = rnn[0], [rnn[1]]
-
-        if axis == 1:
-            outputs = symbol.swapaxes(outputs, dim1=0, dim2=1)
-
-        outputs, _ = _normalize_sequence(length, outputs, layout, merge_outputs)
-
-        return outputs, states
-
-    def unfuse(self):
-        """Unfuse the fused RNN in to a stack of rnn cells.
-
-        Returns
-        -------
-        cell : mxnet.rnn.SequentialRNNCell
-            unfused cell that can be used for stepping, and can run on CPU.
-        """
-        stack = SequentialRNNCell()
-        get_cell = {'rnn_relu': lambda cell_prefix: RNNCell(self._num_hidden,
-                                                            activation='relu',
-                                                            prefix=cell_prefix),
-                    'rnn_tanh': lambda cell_prefix: RNNCell(self._num_hidden,
-                                                            activation='tanh',
-                                                            prefix=cell_prefix),
-                    'lstm': lambda cell_prefix: LSTMCell(self._num_hidden,
-                                                         prefix=cell_prefix),
-                    'gru': lambda cell_prefix: GRUCell(self._num_hidden,
-                                                       prefix=cell_prefix)}[self._mode]
-        for i in range(self._num_layers):
-            if self._bidirectional:
-                stack.add(BidirectionalCell(
-                    get_cell('%sl%d_'%(self._prefix, i)),
-                    get_cell('%sr%d_'%(self._prefix, i)),
-                    output_prefix='%sbi_l%d_'%(self._prefix, i)))
-            else:
-                stack.add(get_cell('%sl%d_'%(self._prefix, i)))
-
-            if self._dropout > 0 and i != self._num_layers - 1:
-                stack.add(DropoutCell(self._dropout, prefix='%s_dropout%d_'%(self._prefix, i)))
-
-        return stack
-
-
-class SequentialRNNCell(BaseRNNCell):
-    """Sequantially stacking multiple RNN cells.
-
-    Parameters
-    ----------
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    """
-    def __init__(self, params=None):
-        super(SequentialRNNCell, self).__init__(prefix='', params=params)
-        self._override_cell_params = params is not None
-        self._cells = []
-
-    def add(self, cell):
-        """Append a cell into the stack.
-
-        Parameters
-        ----------
-        cell : BaseRNNCell
-            The cell to be appended. During unroll, previous cell's output (or raw inputs if
-            no previous cell) is used as the input to this cell.
-        """
-        self._cells.append(cell)
-        if self._override_cell_params:
-            assert cell._own_params, \
-                "Either specify params for SequentialRNNCell " \
-                "or child cells, not both."
-            cell.params._params.update(self.params._params)
-        self.params._params.update(cell.params._params)
-
-    @property
-    def state_info(self):
-        return _cells_state_info(self._cells)
-
-    def begin_state(self, **kwargs): # pylint: disable=arguments-differ
-        assert not self._modified, \
-            "After applying modifier cells (e.g. ZoneoutCell) the base " \
-            "cell cannot be called directly. Call the modifier cell instead."
-        return _cells_begin_state(self._cells, **kwargs)
-
-    def unpack_weights(self, args):
-        return _cells_unpack_weights(self._cells, args)
-
-    def pack_weights(self, args):
-        return _cells_pack_weights(self._cells, args)
-
-    def __call__(self, inputs, states):
-        self._counter += 1
-        next_states = []
-        p = 0
-        for cell in self._cells:
-            assert not isinstance(cell, BidirectionalCell)
-            n = len(cell.state_info)
-            state = states[p:p+n]
-            p += n
-            inputs, state = cell(inputs, state)
-            next_states.append(state)
-        return inputs, sum(next_states, [])
-
-    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
-        self.reset()
-
-        num_cells = len(self._cells)
-        if begin_state is None:
-            begin_state = self.begin_state()
-
-        p = 0
-        next_states = []
-        for i, cell in enumerate(self._cells):
-            n = len(cell.state_info)
-            states = begin_state[p:p+n]
-            p += n
-            inputs, states = cell.unroll(length, inputs=inputs, begin_state=states, layout=layout,
-                                         merge_outputs=None if i < num_cells-1 else merge_outputs)
-            next_states.extend(states)
-
-        return inputs, next_states
-
-
-class DropoutCell(BaseRNNCell):
-    """Apply dropout on input.
-
-    Parameters
-    ----------
-    dropout : float
-        Percentage of elements to drop out, which
-        is 1 - percentage to retain.
-    prefix : str, default ``'dropout_'``
-        Prefix for names of layers
-        (this prefix is also used for names of weights if `params` is None
-        i.e. if `params` are being created and not reused)
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    """
-    def __init__(self, dropout, prefix='dropout_', params=None):
-        super(DropoutCell, self).__init__(prefix, params)
-        assert isinstance(dropout, numeric_types), "dropout probability must be a number"
-        self.dropout = dropout
-
-    @property
-    def state_info(self):
-        return []
-
-    def __call__(self, inputs, states):
-        if self.dropout > 0:
-            inputs = symbol.Dropout(data=inputs, p=self.dropout)
-        return inputs, states
-
-    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
-        self.reset()
-        inputs, _ = _normalize_sequence(length, inputs, layout, merge_outputs)
-        if isinstance(inputs, symbol.Symbol):
-            return self(inputs, [])
-        else:
-            return super(DropoutCell, self).unroll(
-                length, inputs, begin_state=begin_state, layout=layout,
-                merge_outputs=merge_outputs)
-
-
-class ModifierCell(BaseRNNCell):
-    """Base class for modifier cells. A modifier
-    cell takes a base cell, apply modifications
-    on it (e.g. Zoneout), and returns a new cell.
-
-    After applying modifiers the base cell should
-    no longer be called directly. The modifer cell
-    should be used instead.
-    """
-    def __init__(self, base_cell):
-        super(ModifierCell, self).__init__()
-        base_cell._modified = True
-        self.base_cell = base_cell
-
-    @property
-    def params(self):
-        self._own_params = False
-        return self.base_cell.params
-
-    @property
-    def state_info(self):
-        return self.base_cell.state_info
-
-    def begin_state(self, init_sym=symbol.zeros, **kwargs): # pylint: disable=arguments-differ
-        assert not self._modified, \
-            "After applying modifier cells (e.g. DropoutCell) the base " \
-            "cell cannot be called directly. Call the modifier cell instead."
-        self.base_cell._modified = False
-        begin = self.base_cell.begin_state(init_sym, **kwargs)
-        self.base_cell._modified = True
-        return begin
-
-    def unpack_weights(self, args):
-        return self.base_cell.unpack_weights(args)
-
-    def pack_weights(self, args):
-        return self.base_cell.pack_weights(args)
-
-    def __call__(self, inputs, states):
-        raise NotImplementedError
-
-
-class ZoneoutCell(ModifierCell):
-    """Apply Zoneout on base cell.
-
-    Parameters
-    ----------
-    base_cell : BaseRNNCell
-        Cell on whose states to perform zoneout.
-    zoneout_outputs : float, default 0.
-        Fraction of the output that gets dropped out during training time.
-    zoneout_states : float, default 0.
-        Fraction of the states that gets dropped out during training time.
-    """
-    def __init__(self, base_cell, zoneout_outputs=0., zoneout_states=0.):
-        assert not isinstance(base_cell, FusedRNNCell), \
-            "FusedRNNCell doesn't support zoneout. " \
-            "Please unfuse first."
-        assert not isinstance(base_cell, BidirectionalCell), \
-            "BidirectionalCell doesn't support zoneout since it doesn't support step. " \
-            "Please add ZoneoutCell to the cells underneath instead."
-        assert not isinstance(base_cell, SequentialRNNCell) or not base_cell._bidirectional, \
-            "Bidirectional SequentialRNNCell doesn't support zoneout. " \
-            "Please add ZoneoutCell to the cells underneath instead."
-        super(ZoneoutCell, self).__init__(base_cell)
-        self.zoneout_outputs = zoneout_outputs
-        self.zoneout_states = zoneout_states
-        self.prev_output = None
-
-    def reset(self):
-        super(ZoneoutCell, self).reset()
-        self.prev_output = None
-
-    def __call__(self, inputs, states):
-        cell, p_outputs, p_states = self.base_cell, self.zoneout_outputs, self.zoneout_states
-        next_output, next_states = cell(inputs, states)
-        mask = lambda p, like: symbol.Dropout(symbol.ones_like(like), p=p)
-
-        prev_output = self.prev_output if self.prev_output is not None else symbol.zeros((0, 0))
-
-        output = (symbol.where(mask(p_outputs, next_output), next_output, prev_output)
-                  if p_outputs != 0. else next_output)
-        states = ([symbol.where(mask(p_states, new_s), new_s, old_s) for new_s, old_s in
-                   zip(next_states, states)] if p_states != 0. else next_states)
-
-        self.prev_output = output
-
-        return output, states
-
-
-class ResidualCell(ModifierCell):
-    """Adds residual connection as described in Wu et al, 2016
-    (https://arxiv.org/abs/1609.08144).
-
-    Output of the cell is output of the base cell plus input.
-
-    Parameters
-    ----------
-    base_cell : BaseRNNCell
-        Cell on whose outputs to add residual connection.
-    """
-
-    def __init__(self, base_cell):
-        super(ResidualCell, self).__init__(base_cell)
-
-    def __call__(self, inputs, states):
-        output, states = self.base_cell(inputs, states)
-        output = symbol.elemwise_add(output, inputs, name="%s_plus_residual" % output.name)
-        return output, states
-
-    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
-        self.reset()
-
-        self.base_cell._modified = False
-        outputs, states = self.base_cell.unroll(length, inputs=inputs, begin_state=begin_state,
-                                                layout=layout, merge_outputs=merge_outputs)
-        self.base_cell._modified = True
-
-        merge_outputs = isinstance(outputs, symbol.Symbol) if merge_outputs is None else \
-                        merge_outputs
-        inputs, _ = _normalize_sequence(length, inputs, layout, merge_outputs)
-        if merge_outputs:
-            outputs = symbol.elemwise_add(outputs, inputs, name="%s_plus_residual" % outputs.name)
-        else:
-            outputs = [symbol.elemwise_add(output_sym, input_sym,
-                                           name="%s_plus_residual" % output_sym.name)
-                       for output_sym, input_sym in zip(outputs, inputs)]
-
-        return outputs, states
-
-
-class BidirectionalCell(BaseRNNCell):
-    """Bidirectional RNN cell.
-
-    Parameters
-    ----------
-    l_cell : BaseRNNCell
-        cell for forward unrolling
-    r_cell : BaseRNNCell
-        cell for backward unrolling
-    params : RNNParams, default None.
-        Container for weight sharing between cells.
-        A new RNNParams container is created if `params` is None.
-    output_prefix : str, default ``'bi_'``
-        prefix for name of output
-    """
-    def __init__(self, l_cell, r_cell, params=None, output_prefix='bi_'):
-        super(BidirectionalCell, self).__init__('', params=params)
-        self._output_prefix = output_prefix
-        self._override_cell_params = params is not None
-
-        if self._override_cell_params:
-            assert l_cell._own_params and r_cell._own_params, \
-                "Either specify params for BidirectionalCell " \
-                "or child cells, not both."
-            l_cell.params._params.update(self.params._params)
-            r_cell.params._params.update(self.params._params)
-        self.params._params.update(l_cell.params._params)
-        self.params._params.update(r_cell.params._params)
-        self._cells = [l_cell, r_cell]
-
-    def unpack_weights(self, args):
-        return _cells_unpack_weights(self._cells, args)
-
-    def pack_weights(self, args):
-        return _cells_pack_weights(self._cells, args)
-
-    def __call__(self, inputs, states):
-        raise NotImplementedError("Bidirectional cannot be stepped. Please use unroll")
-
-    @property
-    def state_info(self):
-        return _cells_state_info(self._cells)
-
-    def begin_state(self, **kwargs): # pylint: disable=arguments-differ
-        assert not self._modified, \
-            "After applying modifier cells (e.g. DropoutCell) the base " \
-            "cell cannot be called directly. Call the modifier cell instead."
-        return _cells_begin_state(self._cells, **kwargs)
-
-    def unroll(self, length, inputs, begin_state=None, layout='NTC', merge_outputs=None):
-        self.reset()
-
-        inputs, axis = _normalize_sequence(length, inputs, layout, False)
-        if begin_state is None:
-            begin_state = self.begin_state()
-
-        states = begin_state
-        l_cell, r_cell = self._cells
-        l_outputs, l_states = l_cell.unroll(length, inputs=inputs,
-                                            begin_state=states[:len(l_cell.state_info)],
-                                            layout=layout, merge_outputs=merge_outputs)
-        r_outputs, r_states = r_cell.unroll(length,
-                                            inputs=list(reversed(inputs)),
-                                            begin_state=states[len(l_cell.state_info):],
-                                            layout=layout, merge_outputs=merge_outputs)
-
-        if merge_outputs is None:
-            merge_outputs = (isinstance(l_outputs, symbol.Symbol)
-                             and isinstance(r_outputs, symbol.Symbol))
-            if not merge_outputs:
-                if isinstance(l_outputs, symbol.Symbol):
-                    l_outputs = list(symbol.SliceChannel(l_outputs, axis=axis,
-                                                         num_outputs=length, squeeze_axis=1))
-                if isinstance(r_outputs, symbol.Symbol):
-                    r_outputs = list(symbol.SliceChannel(r_outputs, axis=axis,
-                                                         num_outputs=length, squeeze_axis=1))
-
-        if merge_outputs:
-            l_outputs = [l_outputs]
-            r_outputs = [symbol.reverse(r_outputs, axis=axis)]
-        else:
-            r_outputs = list(reversed(r_outputs))
-
-        outputs = [symbol.Concat(l_o, r_o, dim=1+merge_outputs,
-                                 name=('%sout'%(self._output_prefix) if merge_outputs
-                                       else '%st%d'%(self._output_prefix, i)))
-                   for i, l_o, r_o in
-                   zip(range(len(l_outputs)), l_outputs, r_outputs)]
-
-        if merge_outputs:
-            outputs = outputs[0]
-
-        states = [l_states, r_states]
-        return outputs, states
-
-
-class BaseConvRNNCell(BaseRNNCell):
-    """Abstract base class for Convolutional RNN cells"""
-    def __init__(self, input_shape, num_hidden,
-                 h2h_kernel, h2h_dilate,
-                 i2h_kernel, i2h_stride,
-                 i2h_pad, i2h_dilate,
-                 i2h_weight_initializer, h2h_weight_initializer,
-                 i2h_bias_initializer, h2h_bias_initializer,
-                 activation, prefix='', params=None, conv_layout='NCHW'):
-        super(BaseConvRNNCell, self).__init__(prefix=prefix, params=params)
-        # Convolution setting
-        self._h2h_kernel = h2h_kernel
-        assert (self._h2h_kernel[0] % 2 == 1) and (self._h2h_kernel[1] % 2 == 1), \
-            "Only support odd number, get h2h_kernel= %s" % str(h2h_kernel)
-        self._h2h_pad = (h2h_dilate[0] * (h2h_kernel[0] - 1) // 2,
-                         h2h_dilate[1] * (h2h_kernel[1] - 1) // 2)
-        self._h2h_dilate = h2h_dilate
-        self._i2h_kernel = i2h_kernel
-        self._i2h_stride = i2h_stride
-        self._i2h_pad = i2h_pad
-        self._i2h_dilate = i2h_dilate
-
-        self._num_hidden = num_hidden
-        self._input_shape = input_shape
-        self._conv_layout = conv_layout
-        self._activation = activation
-
-        # Infer state shape
-        data = symbol.Variable('data')
-        self._state_shape = symbol.Convolution(data=data,
-                                               num_filter=self._num_hidden,
-                                               kernel=self._i2h_kernel,
-                                               stride=self._i2h_stride,
-                                               pad=self._i2h_pad,
-                                               dilate=self._i2h_dilate,
-                                               layout=conv_layout)
-        self._state_shape = self._state_shape.infer_shape(data=input_shape)[1][0]
-        self._state_shape = (0, ) + self._state_shape[1:]
-
-        # Get params
-        self._iW = self.params.get('i2h_weight', init=i2h_weight_initializer)
-        self._hW = self.params.get('h2h_weight', init=h2h_weight_initializer)
-        self._iB = self.params.get('i2h_bias', init=i2h_bias_initializer)
-        self._hB = self.params.get('h2h_bias', init=h2h_bias_initializer)
-
-    @property
-    def _num_gates(self):
-        return len(self._gate_names)
-
-    @property
-    def state_info(self):
-        return [{'shape': self._state_shape, '__layout__': self._conv_layout},
-                {'shape': self._state_shape, '__layout__': self._conv_layout}]
-
-    def _conv_forward(self, inputs, states, name):
-
-        i2h = symbol.Convolution(name='%si2h'%name,
-                                 data=inputs,
-                                 num_filter=self._num_hidden*self._num_gates,
-                                 kernel=self._i2h_kernel,
-                                 stride=self._i2h_stride,
-                                 pad=self._i2h_pad,
-                                 dilate=self._i2h_dilate,
-                                 weight=self._iW,
-                                 bias=self._iB,
-                                 layout=self._conv_layout)
-
-        h2h = symbol.Convolution(name='%sh2h'%name,
-                                 data=states[0],
-                                 num_filter=self._num_hidden*self._num_gates,
-                                 kernel=self._h2h_kernel,
-                                 dilate=self._h2h_dilate,
-                                 pad=self._h2h_pad,
-                                 stride=(1, 1),
-                                 weight=self._hW,
-                                 bias=self._hB,
-                                 layout=self._conv_layout)
-        return i2h, h2h
-
-    def __call__(self, inputs, states):
-        raise NotImplementedError("BaseConvRNNCell is abstract class for convolutional RNN")
-
-class ConvRNNCell(BaseConvRNNCell):
-    """Convolutional RNN cells
-
-    Parameters
-    ----------
-    input_shape : tuple of int
-        Shape of input in single timestep.
-    num_hidden : int
-        Number of units in output symbol.
-    h2h_kernel : tuple of int, default (3, 3)
-        Kernel of Convolution operator in state-to-state transitions.
-    h2h_dilate : tuple of int, default (1, 1)
-        Dilation of Convolution operator in state-to-state transitions.
-    i2h_kernel : tuple of int, default (3, 3)
-        Kernel of Convolution operator in input-to-state transitions.
-    i2h_stride : tuple of int, default (1, 1)
-        Stride of Convolution operator in input-to-state transitions.
-    i2h_pad : tuple of int, default (1, 1)
-        Pad of Convolution operator in input-to-state transitions.
-    i2h_dilate : tuple of int, default (1, 1)
-        Dilation of Convolution operator in input-to-state transitions.
-    i2h_weight_initializer : str or Initializer
-        Initializer for the input weights matrix, used for the convolution
-        transformation of the inputs.
-    h2h_weight_initializer : str or Initializer
-        Initializer for the recurrent weights matrix, used for the convolution
-        transformation of the recurrent state.
-    i2h_bias_initializer : str or Initializer, default zeros
-        Initializer for the bias vector.
-    h2h_bias_initializer : str or Initializer, default zeros
-        Initializer for the bias vector.
-    activation : str or Symbol,
-        default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
-        Type of activation function.
-    prefix : str, default ``'ConvRNN_'``
-        Prefix for name of layers (and name of weight if params is None).
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    conv_layout : str, , default 'NCHW'
-        Layout of ConvolutionOp
-    """
-    def __init__(self, input_shape, num_hidden,
-                 h2h_kernel=(3, 3), h2h_dilate=(1, 1),
-                 i2h_kernel=(3, 3), i2h_stride=(1, 1),
-                 i2h_pad=(1, 1), i2h_dilate=(1, 1),
-                 i2h_weight_initializer=None, h2h_weight_initializer=None,
-                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',
-                 activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2),
-                 prefix='ConvRNN_', params=None, conv_layout='NCHW'):
-        super(ConvRNNCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden,
-                                          h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,
-                                          i2h_kernel=i2h_kernel, i2h_stride=i2h_stride,
-                                          i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,
-                                          i2h_weight_initializer=i2h_weight_initializer,
-                                          h2h_weight_initializer=h2h_weight_initializer,
-                                          i2h_bias_initializer=i2h_bias_initializer,
-                                          h2h_bias_initializer=h2h_bias_initializer,
-                                          activation=activation, prefix=prefix,
-                                          params=params, conv_layout=conv_layout)
-
-    @property
-    def _gate_names(self):
-        return ('',)
-
-    def __call__(self, inputs, states):
-        self._counter += 1
-        name = '%st%d_'%(self._prefix, self._counter)
-        i2h, h2h = self._conv_forward(inputs, states, name)
-        output = self._get_activation(i2h + h2h, self._activation,
-                                      name='%sout'%name)
-        return output, [output]
-
-    @property
-    def state_info(self):
-        return [{'shape': self._state_shape, '__layout__': self._conv_layout}]
-
-
-class ConvLSTMCell(BaseConvRNNCell):
-    """Convolutional LSTM network cell.
-
-    References
-    ----------
-        Xingjian et al. NIPS2015
-
-    Parameters
-    ----------
-    input_shape : tuple of int
-        Shape of input in single timestep.
-    num_hidden : int
-        Number of units in output symbol.
-    h2h_kernel : tuple of int, default (3, 3)
-        Kernel of Convolution operator in state-to-state transitions.
-    h2h_dilate : tuple of int, default (1, 1)
-        Dilation of Convolution operator in state-to-state transitions.
-    i2h_kernel : tuple of int, default (3, 3)
-        Kernel of Convolution operator in input-to-state transitions.
-    i2h_stride : tuple of int, default (1, 1)
-        Stride of Convolution operator in input-to-state transitions.
-    i2h_pad : tuple of int, default (1, 1)
-        Pad of Convolution operator in input-to-state transitions.
-    i2h_dilate : tuple of int, default (1, 1)
-        Dilation of Convolution operator in input-to-state transitions.
-    i2h_weight_initializer : str or Initializer
-        Initializer for the input weights matrix, used for the convolution
-        transformation of the inputs.
-    h2h_weight_initializer : str or Initializer
-        Initializer for the recurrent weights matrix, used for the convolution
-        transformation of the recurrent state.
-    i2h_bias_initializer : str or Initializer, default zeros
-        Initializer for the bias vector.
-    h2h_bias_initializer : str or Initializer, default zeros
-        Initializer for the bias vector.
-    activation : str or Symbol
-        default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
-        Type of activation function.
-    prefix : str, default ``'ConvLSTM_'``
-        Prefix for name of layers (and name of weight if params is None).
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    conv_layout : str, , default 'NCHW'
-        Layout of ConvolutionOp
-    """
-    def __init__(self, input_shape, num_hidden,
-                 h2h_kernel=(3, 3), h2h_dilate=(1, 1),
-                 i2h_kernel=(3, 3), i2h_stride=(1, 1),
-                 i2h_pad=(1, 1), i2h_dilate=(1, 1),
-                 i2h_weight_initializer=None, h2h_weight_initializer=None,
-                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',
-                 activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2),
-                 prefix='ConvLSTM_', params=None,
-                 conv_layout='NCHW'):
-        super(ConvLSTMCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden,
-                                           h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,
-                                           i2h_kernel=i2h_kernel, i2h_stride=i2h_stride,
-                                           i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,
-                                           i2h_weight_initializer=i2h_weight_initializer,
-                                           h2h_weight_initializer=h2h_weight_initializer,
-                                           i2h_bias_initializer=i2h_bias_initializer,
-                                           h2h_bias_initializer=h2h_bias_initializer,
-                                           activation=activation, prefix=prefix,
-                                           params=params, conv_layout=conv_layout)
-
-    @property
-    def _gate_names(self):
-        return ['_i', '_f', '_c', '_o']
-
-    def __call__(self, inputs, states):
-        self._counter += 1
-        name = '%st%d_'%(self._prefix, self._counter)
-        i2h, h2h = self._conv_forward(inputs, states, name)
-        gates = i2h + h2h
-        slice_gates = symbol.SliceChannel(gates, num_outputs=4, axis=self._conv_layout.find('C'),
-                                          name="%sslice"%name)
-        in_gate = symbol.Activation(slice_gates[0], act_type="sigmoid",
-                                    name='%si'%name)
-        forget_gate = symbol.Activation(slice_gates[1], act_type="sigmoid",
-                                        name='%sf'%name)
-        in_transform = self._get_activation(slice_gates[2], self._activation,
-                                            name='%sc'%name)
-        out_gate = symbol.Activation(slice_gates[3], act_type="sigmoid",
-                                     name='%so'%name)
-        next_c = symbol._internal._plus(forget_gate * states[1], in_gate * in_transform,
-                                        name='%sstate'%name)
-        next_h = symbol._internal._mul(out_gate, self._get_activation(next_c, self._activation),
-                                       name='%sout'%name)
-
-        return next_h, [next_h, next_c]
-
-    @property
-    def state_info(self):
-        return [{'shape': self._state_shape, '__layout__': self._conv_layout},
-                {'shape': self._state_shape, '__layout__': self._conv_layout}]
-
-class ConvGRUCell(BaseConvRNNCell):
-    """Convolutional Gated Rectified Unit (GRU) network cell.
-
-    Parameters
-    ----------
-    input_shape : tuple of int
-        Shape of input in single timestep.
-    num_hidden : int
-        Number of units in output symbol.
-    h2h_kernel : tuple of int, default (3, 3)
-        Kernel of Convolution operator in state-to-state transitions.
-    h2h_dilate : tuple of int, default (1, 1)
-        Dilation of Convolution operator in state-to-state transitions.
-    i2h_kernel : tuple of int, default (3, 3)
-        Kernel of Convolution operator in input-to-state transitions.
-    i2h_stride : tuple of int, default (1, 1)
-        Stride of Convolution operator in input-to-state transitions.
-    i2h_pad : tuple of int, default (1, 1)
-        Pad of Convolution operator in input-to-state transitions.
-    i2h_dilate : tuple of int, default (1, 1)
-        Dilation of Convolution operator in input-to-state transitions.
-    i2h_weight_initializer : str or Initializer
-        Initializer for the input weights matrix, used for the convolution
-        transformation of the inputs.
-    h2h_weight_initializer : str or Initializer
-        Initializer for the recurrent weights matrix, used for the convolution
-        transformation of the recurrent state.
-    i2h_bias_initializer : str or Initializer, default zeros
-        Initializer for the bias vector.
-    h2h_bias_initializer : str or Initializer, default zeros
-        Initializer for the bias vector.
-    activation : str or Symbol,
-        default functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2)
-        Type of activation function.
-    prefix : str, default ``'ConvGRU_'``
-        Prefix for name of layers (and name of weight if params is None).
-    params : RNNParams, default None
-        Container for weight sharing between cells. Created if None.
-    conv_layout : str, , default 'NCHW'
-        Layout of ConvolutionOp
-    """
-    def __init__(self, input_shape, num_hidden,
-                 h2h_kernel=(3, 3), h2h_dilate=(1, 1),
-                 i2h_kernel=(3, 3), i2h_stride=(1, 1),
-                 i2h_pad=(1, 1), i2h_dilate=(1, 1),
-                 i2h_weight_initializer=None, h2h_weight_initializer=None,
-                 i2h_bias_initializer='zeros', h2h_bias_initializer='zeros',
-                 activation=functools.partial(symbol.LeakyReLU, act_type='leaky', slope=0.2),
-                 prefix='ConvGRU_', params=None, conv_layout='NCHW'):
-        super(ConvGRUCell, self).__init__(input_shape=input_shape, num_hidden=num_hidden,
-                                          h2h_kernel=h2h_kernel, h2h_dilate=h2h_dilate,
-                                          i2h_kernel=i2h_kernel, i2h_stride=i2h_stride,
-                                          i2h_pad=i2h_pad, i2h_dilate=i2h_dilate,
-                                          i2h_weight_initializer=i2h_weight_initializer,
-                                          h2h_weight_initializer=h2h_weight_initializer,
-                                          i2h_bias_initializer=i2h_bias_initializer,
-                                          h2h_bias_initializer=h2h_bias_initializer,
-                                          activation=activation, prefix=prefix,
-                                          params=params, conv_layout=conv_layout)
-
-    @property
-    def _gate_names(self):
-        return ['_r', '_z', '_o']
-
-    @property
-    def state_info(self):
-        return [{'shape': self._state_shape, '__layout__': self._conv_layout}]
-
-    def __call__(self, inputs, states):
-        self._counter += 1
-        seq_idx = self._counter
-        name = '%st%d_' % (self._prefix, seq_idx)
-        i2h, h2h = self._conv_forward(inputs, states, name)
-        # pylint: disable=unbalanced-tuple-unpacking
-        i2h_r, i2h_z, i2h = symbol.SliceChannel(i2h, num_outputs=3, name="%s_i2h_slice" % name)
-        h2h_r, h2h_z, h2h = symbol.SliceChannel(h2h, num_outputs=3, name="%s_h2h_slice" % name)
-
-        reset_gate = symbol.Activation(i2h_r + h2h_r, act_type="sigmoid",
-                                       name="%s_r_act" % name)
-        update_gate = symbol.Activation(i2h_z + h2h_z, act_type="sigmoid",
-                                        name="%s_z_act" % name)
-
-        next_h_tmp = self._get_activation(i2h + reset_gate * h2h, self._activation,
-                                          name="%s_h_act" % name)
-
-        next_h = symbol._internal._plus((1. - update_gate) * next_h_tmp, update_gate * states[0],
-                                        name='%sout' % name)
-
-        return next_h, [next_h]
diff --git a/tests/python/gpu/test_contrib_amp.py b/tests/python/gpu/test_contrib_amp.py
index 4d943bd..d1d9162 100644
--- a/tests/python/gpu/test_contrib_amp.py
+++ b/tests/python/gpu/test_contrib_amp.py
@@ -33,7 +33,6 @@ curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.insert(0, os.path.join(curr_path, '../unittest'))
 from common import with_seed, teardown_module, assert_raises_cudnn_not_satisfied
 sys.path.insert(0, os.path.join(curr_path, '../train'))
-from test_bucketing import train_model
 set_default_context(mx.gpu(0))
 
 @pytest.fixture()
@@ -314,44 +313,10 @@ def test_amp_conversion(amp_tests):
         assert params["stage2_unit1_conv2_weight"].dtype == np.float16
 
 
-    def check_amp_convert_bucketing_module():
-        model = train_model(context=mx.current_context())
-        result_model = amp.convert_bucketing_module(model)
-        val_sent = []
-        batch_size = 128
-        invalid_label = -1
-        num_sentence = 1000
-        buckets = [5, 10, 20, 30, 40]
-        len_vocab = 50
-
-        for _ in range(num_sentence):
-            len_sentence = randint(6, max(buckets)-1) # leave out the two last buckets empty
-            val_sentence = []
-            for _ in range(len_sentence):
-                val_sentence.append(randint(1, len_vocab))
-            val_sent.append(val_sentence)
-
-        data_val =  mx.rnn.BucketSentenceIter(val_sent, batch_size, buckets=buckets,
-                                     invalid_label=invalid_label)
-        result_model.bind(data_val.provide_data, data_val.provide_label, for_training=False)
-        result_model.score(data_val, mx.gluon.metric.Perplexity(invalid_label),
-                           batch_end_callback=mx.callback.Speedometer(batch_size, 1))
-
-        # AMP conversion with cast_optional_params set to true
-        # Flaky test when cast_optional_params set to True : https://github.com/apache/incubator-mxnet/issues/16030
-        '''
-        result_model = amp.convert_bucketing_module(model, cast_optional_params=True)
-        result_model.bind(data_val.provide_data, data_val.provide_label, for_training=False)
-        result_model.score(data_val, mx.gluon.metric.Perplexity(invalid_label),
-                           batch_end_callback=mx.callback.Speedometer(batch_size, 1))
-        '''
-
-
     with mx.Context(mx.gpu(0)):
         check_amp_convert_symbol()
         check_amp_convert_model()
         check_amp_convert_hybrid_block()
-        check_amp_convert_bucketing_module()
 
 @with_seed()
 @pytest.mark.skip(reason='Error during waitall(). Tracked in #18099')
diff --git a/tests/python/gpu/test_operator_gpu.py b/tests/python/gpu/test_operator_gpu.py
index e92c296..b6d0011 100644
--- a/tests/python/gpu/test_operator_gpu.py
+++ b/tests/python/gpu/test_operator_gpu.py
@@ -38,7 +38,6 @@ from test_numpy_interoperability import *
 from test_optimizer import *
 from test_random import *
 from test_exc_handling import *
-#from test_rnn import *
 from test_sparse_ndarray import *
 from test_sparse_operator import *
 from test_ndarray import *
@@ -1742,90 +1741,6 @@ def check_rnn_consistency(cell1, cell2):
     mx.test_utils.assert_allclose(mod1.get_outputs()[0], mod2.get_outputs()[0], rtol=1e-2, atol=1e-4)
 
 @with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnn():
-    fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='rnn_relu', prefix='')
-
-    stack = mx.rnn.SequentialRNNCell()
-    stack.add(mx.rnn.RNNCell(100, activation='relu', prefix='l0_'))
-    stack.add(mx.rnn.RNNCell(100, activation='relu', prefix='l1_'))
-
-    check_rnn_consistency(fused, stack)
-    check_rnn_consistency(stack, fused)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-def test_lstm_forget_bias():
-    forget_bias = 2.0
-    fused = mx.rnn.FusedRNNCell(10, forget_bias=forget_bias, num_layers=2, mode='lstm', prefix='')
-
-    dshape = (32, 1, 20)
-    data = mx.sym.Variable('data')
-
-    sym, _ = fused.unroll(1, data, merge_outputs=True)
-    mod = mx.mod.Module(sym, label_names=None, context=mx.gpu(0))
-    mod.bind(data_shapes=[('data', dshape)], label_shapes=None)
-
-    mod.init_params()
-
-    args, auxs = mod.get_params()
-    args = fused.unpack_weights(args)
-
-    bias_name = next(x for x in args if x.endswith('f_bias'))
-    expected_bias = forget_bias * np.ones(10, )
-    mx.test_utils.assert_allclose(args[bias_name], expected_bias)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_gru():
-    fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='')
-
-    stack = mx.rnn.SequentialRNNCell()
-    stack.add(mx.rnn.GRUCell(100, prefix='l0_'))
-    stack.add(mx.rnn.GRUCell(100, prefix='l1_'))
-
-    check_rnn_consistency(fused, stack)
-    check_rnn_consistency(stack, fused)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_bidirectional():
-    fused = mx.rnn.FusedRNNCell(100, num_layers=2, mode='gru', prefix='',
-            bidirectional=True)
-
-    stack = mx.rnn.SequentialRNNCell()
-    stack.add(mx.rnn.BidirectionalCell(
-                mx.rnn.GRUCell(100, prefix='l0_'),
-                mx.rnn.GRUCell(100, prefix='r0_'),
-                output_prefix='bi_gru_0_'))
-    stack.add(mx.rnn.BidirectionalCell(
-                mx.rnn.GRUCell(100, prefix='l1_'),
-                mx.rnn.GRUCell(100, prefix='r1_'),
-                output_prefix='bi_gru_1_'))
-
-    check_rnn_consistency(fused, stack)
-    check_rnn_consistency(stack, fused)
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-def test_unfuse():
-    for mode in ['rnn_tanh', 'rnn_relu', 'lstm', 'gru']:
-        fused = mx.rnn.FusedRNNCell(
-            100, num_layers=2, mode=mode,
-            prefix='test_%s'%mode,
-            bidirectional=True,
-            dropout=0.5)
-
-        stack = fused.unfuse()
-
-        check_rnn_consistency(fused, stack)
-        check_rnn_consistency(stack, fused)
-
-
-@with_seed()
 @pytest.mark.serial
 def test_psroipooling_with_type():
     arg_params = {
@@ -2036,29 +1951,6 @@ def test_deformable_convolution_options():
     check_consistency(sym, ctx_list, scale=0.1, tol=tol)
 
 
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_residual_fused():
-    cell = mx.rnn.ResidualCell(
-            mx.rnn.FusedRNNCell(50, num_layers=3, mode='lstm',
-                               prefix='rnn_', dropout=0.5))
-
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)]
-    outputs, _ = cell.unroll(2, inputs, merge_outputs=None)
-    assert sorted(cell.params._params.keys()) == \
-           ['rnn_parameters']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50))
-    assert outs == [(10, 2, 50)]
-    outputs = outputs.eval(ctx=mx.gpu(0),
-                           rnn_t0_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5,
-                           rnn_t1_data=mx.nd.ones((10, 50), ctx=mx.gpu(0))+5,
-                           rnn_parameters=mx.nd.zeros((61200,), ctx=mx.gpu(0)))
-    expected_outputs = np.ones((10, 2, 50))+5
-    assert np.array_equal(outputs[0].asnumpy(), expected_outputs)
-
-
 def check_rnn_layer(layer):
     layer.collect_params().initialize(ctx=[mx.cpu(0), mx.gpu(0)])
     with mx.gpu(0):
diff --git a/tests/python/quantization/test_quantization.py b/tests/python/quantization/test_quantization.py
index 71f044f..5034d90 100644
--- a/tests/python/quantization/test_quantization.py
+++ b/tests/python/quantization/test_quantization.py
@@ -957,169 +957,6 @@ def test_quantize_model():
     for qdtype in ['int8', 'uint8']:
         check_quantize_model(qdtype)
 
-@with_seed()
-def test_quantize_model_with_forward():
-    def check_quantize_model(qdtype):
-        if is_test_for_native_cpu():
-            print('skipped testing test_quantize_model_with_forward for native cpu since it is not supported yet')
-            return
-        elif qdtype == 'uint8' and is_test_for_gpu():
-            print('skipped testing test_quantize_model_with_forward for gpu uint8 since it is not supported yet')
-            return
-
-        def check_params(params, qparams, qsym=None):
-            if qsym is None:
-                assert len(params) == len(qparams)
-                for k, v in params.items():
-                    assert k in qparams
-                    assert same(v.asnumpy(), qparams[k].asnumpy())
-            else:
-                qparams_ground_truth = mx.contrib.quant._quantize_params(qsym, params, th_dict = {})
-                assert len(qparams) == len(qparams_ground_truth)
-                for k, v in qparams_ground_truth.items():
-                    assert k in qparams
-                    assert same(v.asnumpy(), qparams[k].asnumpy())
-
-        def check_qsym_calibrated(qsym):
-            attrs = qsym.attr_dict()
-            for k, v in attrs.items():
-                if k.find('requantize_') != -1:
-                    assert 'min_calib_range' in v
-                    assert 'max_calib_range' in v
-
-        def check_qsym_qdtype(qsym, qdtype):
-            attrs = qsym.attr_dict()
-            for k, v in attrs.items():
-                if k.find('_quantize') != -1:
-                    assert 'out_type' in v
-                    assert v['out_type'] == qdtype
-
-        def check_qsym_forward(qsym, qarg_params, qaux_params, data_shape, label_shape=None):
-            if label_shape is None:
-                mod = mx.mod.Module(symbol=qsym, label_names=None, context=mx.current_context())
-                mod.bind(for_training=False,
-                         data_shapes=[('data', data_shape)])
-            else:
-                mod = mx.mod.Module(symbol=qsym, context=mx.current_context())
-                mod.bind(for_training=False,
-                         data_shapes=[('data', data_shape)],
-                         label_shapes=[('softmax_label', label_shape)])
-            mod.set_params(qarg_params, qaux_params)
-            data = [mx.random.uniform(-1.0, 1.0, shape=shape) for _, shape in mod.data_shapes]
-            batch = mx.io.DataBatch(data, [])
-            mod.forward(batch, is_train=False)
-            for output in mod.get_outputs():
-                output.wait_to_read()
-
-        batch_size = 4
-        length = batch_size  # specify num of outputs from split op
-        sym_list = []
-        name_list = []
-        dshape_list = []
-        lshape_list = []
-
-        # sym 1
-        sym_list.append(get_fp32_residual())
-        name_list.append('sym1')
-        dshape_list.append((batch_size, 4, 10, 10))
-        lshape_list.append((batch_size, 10))
-
-        # sym 2
-        sym_list.append(get_fp32_sym_with_multiple_outputs(length))
-        name_list.append('sym2')
-        dshape_list.append((length, 4, 4, 10, 10))
-        lshape_list.append((length, 10))
-
-        data = mx.sym.Variable('data')
-        # sym 3
-        sym_list.append(mx.sym.Convolution(data, kernel=(1, 1), num_filter=16, name='conv0'))
-        name_list.append('sym3')
-        dshape_list.append((batch_size, 4, 10, 10))
-        lshape_list.append(None)
-
-        # sym 4
-        cell = mx.rnn.LSTMCell(num_hidden=64)
-        outputs, _ = cell.unroll(length, data)
-        sym_list.append(mx.sym.Group(outputs))
-        name_list.append('sym4')
-        dshape_list.append((batch_size, length, 32))
-        lshape_list.append(None)
-
-        for s, dshape, lshape, name in zip(sym_list, dshape_list, lshape_list, name_list):
-            if qdtype == 'int8' and name in ['sym1','sym2','sym3']:
-                print('mkldnn_quantized_conv op only supports uint8 as input type, skip test with int8.')
-                continue
-            if qdtype == 'uint8' and name in ['sym1']:
-                print('mkldnn_quantized_bn doesn\'t support calib_mode=None')
-                continue
-            if lshape is None:
-                mod = Module(symbol=s, label_names=None)
-                mod.bind(for_training=False,
-                         data_shapes=[('data', dshape)])
-            else:
-                mod = Module(symbol=s)
-                mod.bind(for_training=False,
-                         data_shapes=[('data', dshape)],
-                         label_shapes=[('softmax_label', lshape)])
-
-            mod.init_params()
-            arg_params, aux_params = mod.get_params()
-
-            excluded_sym_names = []
-            excluded_op_names = []
-            # sym3/sym4 doesn't have such layers
-            if name not in ['sym3', 'sym4']:
-                excluded_names = []
-                if mx.current_context() == mx.cpu():
-                   excluded_op_names += ['FullyConnected']
-                   excluded_names += ['conv1']
-                excluded_names += ['concat']
-
-                optional_names = ['pool0']
-                for skip_optional_names in [False, True]:
-                    exclude_sym_names = []
-                    if skip_optional_names:
-                        excluded_sym_names = excluded_names
-                    else:
-                        excluded_sym_names = excluded_names + optional_names
-            if name == 'sym4':
-                excluded_op_names += ['elemwise_add', 'elemwise_mul']
-
-            qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=s,
-                                                                             arg_params=arg_params,
-                                                                             aux_params=aux_params,
-                                                                             excluded_sym_names=excluded_sym_names,
-                                                                             excluded_op_names=excluded_op_names,
-                                                                             ctx=mx.current_context(),
-                                                                             quantized_dtype=qdtype,
-                                                                             calib_mode='none',
-                                                                             quantize_mode='full')
-            check_params(arg_params, qarg_params, qsym)
-            check_params(aux_params, qaux_params)
-            check_qsym_forward(qsym, qarg_params, qaux_params, dshape, lshape)
-
-            calib_data = mx.nd.random.uniform(shape=dshape)
-            calib_data = NDArrayIter(data=calib_data, batch_size=batch_size)
-            calib_data = DummyIter(calib_data)
-            qsym, qarg_params, qaux_params = mx.contrib.quant.quantize_model(sym=s,
-                                                                             arg_params=arg_params,
-                                                                             aux_params=aux_params,
-                                                                             excluded_sym_names=excluded_sym_names,
-                                                                             excluded_op_names=excluded_op_names,
-                                                                             ctx=mx.current_context(),
-                                                                             quantized_dtype=qdtype,
-                                                                             calib_mode='naive',
-                                                                             calib_data=calib_data,
-                                                                             num_calib_examples=20,
-                                                                             quantize_mode='full')
-            check_params(arg_params, qarg_params, qsym)
-            check_params(aux_params, qaux_params)
-            check_qsym_calibrated(qsym)
-            check_qsym_qdtype(qsym, qdtype)
-            check_qsym_forward(qsym, qarg_params, qaux_params, dshape, lshape)
-
-    for qdtype in ['int8', 'uint8']:
-        check_quantize_model(qdtype)
 
 @with_seed()
 def test_quantize_gluon_with_forward():
diff --git a/tests/python/train/test_bucketing.py b/tests/python/train/test_bucketing.py
deleted file mode 100644
index 6734a5d..0000000
--- a/tests/python/train/test_bucketing.py
+++ /dev/null
@@ -1,125 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-# pylint: skip-file
-import numpy as np
-import mxnet as mx
-import random
-from random import randint
-from mxnet.contrib.amp import amp
-
-import pytest
-
-
-def prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence):
-    train_sent = []
-    val_sent = []
-
-    for _ in range(num_sentence):
-        len_sentence = randint(6, max(buckets)-1) # leave out the two last buckets empty
-        train_sentence = []
-        val_sentence = []
-        for _ in range(len_sentence):
-            train_sentence.append(randint(1, len_vocab))
-            val_sentence.append(randint(1, len_vocab))
-        train_sent.append(train_sentence)
-        val_sent.append(val_sentence)
-
-    data_train = mx.rnn.BucketSentenceIter(train_sent, batch_size, buckets=buckets,
-                                   invalid_label=invalid_label)
-    data_val =  mx.rnn.BucketSentenceIter(val_sent, batch_size, buckets=buckets,
-                                 invalid_label=invalid_label)
-
-    return (data_train, data_val)
-
-
-def train_model(context=mx.cpu()):
-    import logging
-    head = '%(asctime)-15s %(message)s'
-    logging.basicConfig(level=logging.DEBUG, format=head)
-    console = logging.StreamHandler()
-    console.setLevel(logging.DEBUG)
-    logging.getLogger('').addHandler(console)
-
-    batch_size = 128
-    num_epochs = 5
-    num_hidden = 25
-    num_embed = 25
-    num_layers = 2
-    len_vocab = 50
-    buckets = [5, 10, 20, 30, 40]
-
-    invalid_label = -1
-    num_sentence = 1000
-
-    data_train, data_val = prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence)
-
-    stack = mx.rnn.SequentialRNNCell()
-    for i in range(num_layers):
-        stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i))
-
-    def sym_gen(seq_len):
-        data = mx.sym.Variable('data')
-        label = mx.sym.Variable('softmax_label')
-        embed = mx.sym.Embedding(data=data, input_dim=len_vocab,
-                                 output_dim=num_embed, name='embed')
-
-        stack.reset()
-        outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
-        pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
-        pred = mx.sym.FullyConnected(data=pred, num_hidden=len_vocab, name='pred')
-
-        label = mx.sym.Reshape(label, shape=(-1,))
-        loss = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-
-        return loss, ('data',), ('softmax_label',)
-
-    contexts = context
-
-    model = mx.mod.BucketingModule(
-        sym_gen=sym_gen,
-        default_bucket_key=data_train.default_bucket_key,
-        context=contexts)
-
-    logging.info('Begin fit...')
-    model.fit(
-        train_data=data_train,
-        eval_data=data_val,
-        eval_metric=mx.gluon.metric.Perplexity(invalid_label), # Use Perplexity for multiclass classification.
-        kvstore='device',
-        optimizer='sgd',
-        optimizer_params={'learning_rate': 0.01,
-                          'momentum': 0,
-                          'wd': 0.00001},
-        initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
-        num_epoch=num_epochs,
-        batch_end_callback=mx.callback.Speedometer(batch_size, 50))
-    logging.info('Finished fit...')
-    return model
-
-
-@pytest.mark.garbage_expected
-def test_bucket_module():
-    # This test forecasts random sequence of words to check bucketing.
-    # We cannot guarantee the accuracy of such an impossible task, and comments out the following line.
-    # assert model.score(data_val, mx.gluon.metric.MSE())[0][1] < 350, "High mean square error."
-    model = train_model()
-
-
-if __name__ == "__main__":
-    test_bucket_module()
diff --git a/tests/python/unittest/test_contrib_control_flow.py b/tests/python/unittest/test_contrib_control_flow.py
index b703402..3c8950d 100644
--- a/tests/python/unittest/test_contrib_control_flow.py
+++ b/tests/python/unittest/test_contrib_control_flow.py
@@ -899,101 +899,6 @@ def test_while_loop_nested():
             assert_almost_equal(x, y, rtol=1e-3, atol=1e-3)
 
 
-@with_seed()
-def test_while_loop_rnn():
-    def _array(shape):
-        return mx.nd.random.uniform(-1.0, 1.0, shape=shape)
-
-    cell_types = [mx.rnn.LSTMCell]
-    num_params = [2]
-
-    batch_size = 2
-    hidden_dim = 3
-    input_dim = 4
-    seq_len = 3
-
-    for cell, n_param in zip(cell_types, num_params):
-        # using while_loop
-        params = mx.rnn.RNNParams()
-        data = mx.sym.var("data")
-        iter_i = mx.sym.var("i")
-        def _cond(*states):
-            i = states[0]
-            return i < seq_len
-        def _func(*states):
-            i = states[0]
-            states = states[1:]
-            in_ = data.take(i).squeeze(axis=0)
-            rnn = cell(hidden_dim, prefix='', params=params)
-            next_hidden, next_states = rnn(in_, states)
-            return [next_hidden], [i + 1] + list(next_states)
-        states = [mx.sym.var("s_" + str(i)) for i in range(n_param)]
-        result = mx.sym.contrib.while_loop(
-                    cond=_cond,
-                    func=_func,
-                    loop_vars=[iter_i] + states,
-                    max_iterations=seq_len
-                )
-        result = mx.sym.Group(result[0] + result[1][1: ])
-        rnn_inputs = result.list_inputs()
-        args = {
-            "i": mx.nd.zeros([1]),
-            "data": _array((seq_len, batch_size, input_dim)),
-            "i2h_weight": _array((input_dim * hidden_dim, input_dim)),
-            "i2h_bias": _array((input_dim * hidden_dim, )),
-            "s_0": _array((batch_size, hidden_dim)),
-            "h2h_weight": _array((input_dim * hidden_dim, seq_len)),
-            "h2h_bias": _array((input_dim * hidden_dim, )),
-            "s_1": _array((batch_size, hidden_dim)),
-        }
-        args_grad = {
-            "i": _array([1]),
-            "data": _array((seq_len, batch_size, input_dim)),
-            "i2h_weight": _array((input_dim * hidden_dim, input_dim)),
-            "i2h_bias": _array((input_dim * hidden_dim, )),
-            "s_0": _array((batch_size, hidden_dim)),
-            "h2h_weight": _array((input_dim * hidden_dim, seq_len)),
-            "h2h_bias": _array((input_dim * hidden_dim, )),
-            "s_1": _array((batch_size, hidden_dim)),
-        }
-        e_1 = result.bind(ctx=default_context(),
-            args={name: array.copy() for name, array in args.items()},
-            args_grad={name: array.copy() for name, array in args_grad.items() if name != "i"},
-        )
-        # using unrolled rnn
-        rnn = cell(hidden_dim, prefix='')
-        unroll_outs = []
-        for inputs in mx.sym.split(data, num_outputs=seq_len, axis=0, squeeze_axis=True):
-            h, states = rnn(inputs, states)
-            unroll_outs.append(mx.sym.expand_dims(h, axis=0))
-        unroll_outs = _as_list(mx.sym.concat(*unroll_outs, dim=0))
-        unroll_outs.extend(states)
-        result = mx.sym.Group(unroll_outs)
-        e_2 = result.bind(ctx=default_context(),
-            args={name: array.copy() for name, array in args.items() if name != "i"},
-            args_grad={name: array.copy() for name, array in args_grad.items() if name != "i"},
-        )
-        for case_id in range(100):
-            args = {name: array.copy() for name, array in args.items()}
-            e_1.forward(is_train=True, **args)
-            out_grads = [_array(arr.shape) for arr in e_1.outputs]
-            e_1.backward(out_grads)
-            args = {name: array.copy() for name, array in args.items() if name != "i"}
-            e_2.forward(is_train=True, **args)
-            e_2.backward(out_grads)
-            assert len(e_1.outputs) == len(e_2.outputs)
-            for x, y in zip(e_1.outputs, e_2.outputs):
-                x = x.asnumpy()
-                y = y.asnumpy()
-                assert_almost_equal(x, y, rtol=1e-3, atol=1e-3)
-            grad_keys = list(e_2.grad_dict.keys())
-            e_1_grad = [e_1.grad_dict[x] for x in grad_keys]
-            e_2_grad = [e_2.grad_dict[x] for x in grad_keys]
-            for x, y in zip(e_1_grad, e_2_grad):
-                x = x.asnumpy()
-                y = y.asnumpy()
-                assert_almost_equal(x, y, rtol=1e-3, atol=1e-3)
-
 def _verify_cond(cond_func, then_func, else_func, input_var_shapes, free_var_shapes, is_train):
 
     def _create_symbol(prefix, i):
@@ -1586,104 +1491,6 @@ def test_foreach_nested():
     assert_almost_equal(state.grad.asnumpy(), state_grad.asnumpy(), rtol=1e-3, atol=1e-3)
 
 
-def check_foreach_rnn(cell_type, num_states):
-    data = mx.sym.var("data")
-    params = mx.rnn.RNNParams()
-    hidden_dim = 4
-    input_dim = 5
-    seq_len = 2
-    batch_size = 2
-
-    # This tests foreach with accumulation sum.
-    def step(in1, states):
-        rnn = cell_type(hidden_dim, prefix='', params=params)
-        next_h, states = rnn(in1, states)
-        return (next_h, states)
-
-    def sym_group(out):
-        if (isinstance(out[0], mx.sym.Symbol)):
-            ret = [out[0]]
-        else:
-            ret = out[0]
-        ret.extend(out[1])
-        return mx.sym.Group(ret)
-
-    rnn = cell_type(hidden_dim, prefix='', params=params)
-    if num_states == 2:
-        init_states = [mx.sym.var("h"), mx.sym.var("c")]
-    else:
-        init_states = [mx.sym.var("h")]
-    out = mx.sym.contrib.foreach(step, data, init_states)
-    out = sym_group(out)
-    arg_shapes, out_shapes, aux_shapes = out.infer_shape(data=(seq_len, batch_size, input_dim),
-            h=(batch_size, hidden_dim))
-    rnn_inputs = out.list_inputs()
-
-    # Inputs
-    args1 = {name:mx.nd.random.uniform(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
-    args2 = copy.deepcopy(args1)
-    # gradients for the backward of the foreach symbol
-    args_grad1 = {name:mx.nd.empty(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
-    # gradients for the backward of the unrolled symbol.
-    args_grad2 = {name:mx.nd.empty(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
-
-    # Symbol of running LSTM with foreach.
-    out = mx.sym.contrib.foreach(step, data, init_states)
-    out = sym_group(out)
-    js_1 = out.tojson()
-    out = mx.sym.load_json(js_1)
-    js_2 = out.tojson()
-    assert js_1 == js_2
-    e1 = out.bind(ctx=default_context(), args=args1, args_grad=args_grad1)
-
-    # Symbol of running unrolled LSTM.
-    lstm = cell_type(hidden_dim, prefix='')
-    unroll_outs = []
-    states = init_states
-    for inputs in mx.sym.split(data, num_outputs=seq_len, axis=0, squeeze_axis=True):
-        h, states = lstm(inputs, states)
-        unroll_outs.append(mx.sym.expand_dims(h, axis=0))
-    unroll_outs = _as_list(mx.sym.concat(*unroll_outs, dim=0))
-    unroll_outs.extend(states)
-    out = mx.sym.Group(unroll_outs)
-    js_1 = out.tojson()
-    out = mx.sym.load_json(js_1)
-    js_2 = out.tojson()
-    assert js_1 == js_2
-    e2 = out.bind(ctx=default_context(), args=args2, args_grad=args_grad2)
-
-    for i in range(5):
-        out_grads = []
-        for arr in e1.outputs:
-            out_grads.append(mx.nd.random.uniform(-10, 10, arr.shape))
-
-        args = {name:mx.nd.random.uniform(shape=arg_shapes[i]) for i, name in enumerate(rnn_inputs)}
-
-        e1.forward(is_train=True, **args)
-        outputs1 = e1.outputs
-        e1.backward(out_grads)
-
-        e2.forward(is_train=True, **args)
-        outputs2 = e2.outputs
-        e2.backward(out_grads)
-
-        for i in range(len(outputs2)):
-            assert_almost_equal(outputs1[i].asnumpy(), outputs2[i].asnumpy(),
-                    rtol=1e-3, atol=1e-3)
-        input_names = out.list_inputs()
-        for i in range(len(e1.grad_arrays)):
-            name = input_names[i]
-            assert_almost_equal(args_grad1[name].asnumpy(), args_grad2[name].asnumpy(),
-                    rtol=1e-3, atol=1e-3)
-
-
-@with_seed()
-def test_foreach_rnn():
-    cell_types = [(mx.rnn.LSTMCell, 2), (mx.rnn.RNNCell, 1), (mx.rnn.GRUCell, 1)]
-    for cell_type, num_states in cell_types:
-        check_foreach_rnn(cell_type, num_states)
-
-
 @with_seed()
 def test_cut_subgraph_foreach():
     class TestLayer(gluon.HybridBlock):
diff --git a/tests/python/unittest/test_module.py b/tests/python/unittest/test_module.py
index bfb72a4..b7f12ff 100644
--- a/tests/python/unittest/test_module.py
+++ b/tests/python/unittest/test_module.py
@@ -26,7 +26,6 @@ from common import setup_module, with_seed, assertRaises, teardown_module
 from collections import namedtuple
 curr_path = os.path.dirname(os.path.abspath(os.path.expanduser(__file__)))
 sys.path.insert(0, os.path.join(curr_path, "../train"))
-from test_bucketing import train_model, prepare_bucketing_data
 import pytest
 
 
@@ -212,76 +211,6 @@ def test_save_load(ctx, get_updater, tmpdir):
 
 
 @with_seed()
-@pytest.mark.garbage_expected
-def test_bucketing_save_load(tmpdir):
-    previous_update_on_kvstore = os.getenv('MXNET_UPDATE_ON_KVSTORE', "1")
-    os.putenv('MXNET_UPDATE_ON_KVSTORE', '1')
-    def dict_equ(a, b):
-        assert set(a) == set(b)
-        for k in a:
-            assert (a[k].asnumpy() == b[k].asnumpy()).all()
-
-
-    len_vocab = 50
-    num_embed = 25
-    num_epochs = 5
-    batch_size = 128
-    num_layers = 2
-    num_hidden = 25
-    buckets = [5, 10, 20, 30, 40]
-    invalid_label = -1
-    num_sentence=1000
-
-    stack = mx.rnn.SequentialRNNCell()
-    for i in range(num_layers):
-        stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i))
-
-    def sym_gen(seq_len):
-        data = mx.sym.Variable('data')
-        label = mx.sym.Variable('softmax_label')
-        embed = mx.sym.Embedding(data=data, input_dim=len_vocab,
-                                 output_dim=num_embed, name='embed')
-        stack.reset()
-        outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
-        pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
-        pred = mx.sym.FullyConnected(data=pred, num_hidden=len_vocab, name='pred')
-
-        label = mx.sym.Reshape(label, shape=(-1,))
-        loss = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-
-        return loss, ('data',), ('softmax_label',)
-
-    path = str(tmpdir.join('test'))
-
-    model = train_model(context=mx.current_context())
-    model.save_checkpoint(path, 0)
-    data_train, data_val = prepare_bucketing_data(buckets, len_vocab, batch_size, invalid_label, num_sentence)
-    mod2 = mx.mod.BucketingModule.load(path, 0, sym_gen=sym_gen,
-                                       default_bucket_key=data_train.default_bucket_key)
-
-    mod2.bind(data_shapes=data_train.provide_data,
-              label_shapes=data_train.provide_label)
-
-    for bucket_key in model._buckets.keys():
-        dict_equ(model._buckets[model._default_bucket_key].get_params()[0],
-                 mod2._buckets[mod2._default_bucket_key].get_params()[0])
-    mod2.fit(
-        train_data=data_train,
-        eval_data=data_val,
-        eval_metric=mx.gluon.metric.Perplexity(invalid_label), # Use Perplexity for multiclass classification.
-        kvstore='device',
-        optimizer='sgd',
-        optimizer_params={'learning_rate': 0.01,
-                          'momentum': 0,
-                          'wd': 0.00001},
-        initializer=mx.init.Xavier(factor_type="in", magnitude=2.34),
-        num_epoch=num_epochs,
-        batch_end_callback=mx.callback.Speedometer(batch_size, 50))
-    os.putenv('MXNET_UPDATE_ON_KVSTORE', previous_update_on_kvstore)
-
-
-@with_seed()
 def test_module_reshape():
     data = mx.sym.Variable('data')
     sym = mx.sym.FullyConnected(data, num_hidden=20, name='fc')
@@ -309,93 +238,6 @@ def test_module_reshape():
     assert (mod.get_params()[0]['fc_bias'].asnumpy() == -3).all()
 
 
-@with_seed()
-def test_module_states():
-    stack = mx.rnn.SequentialRNNCell()
-    for i in range(2):
-        stack.add(mx.rnn.LSTMCell(num_hidden=20, prefix='lstm_l%d_'%i))
-    begin_state = stack.begin_state(func=mx.sym.Variable)
-    _, states = stack.unroll(10, begin_state=begin_state, inputs=mx.sym.Variable('data'))
-
-    state_names = [i.name for i in begin_state]
-    mod = mx.mod.Module(mx.sym.Group(states), context=[mx.cpu(0), mx.cpu(1)],
-                        label_names=None, state_names=state_names)
-    mod.bind(data_shapes=[('data', (5, 10))], label_shapes=None, for_training=False)
-    mod.init_params()
-    batch = mx.io.DataBatch(data=[mx.nd.zeros((5, 10))], label=[])
-
-    mod.set_states(value=1)
-    mod.forward(batch)
-    out = mod.get_outputs(merge_multi_context=False)
-    out1 = mod.get_outputs(merge_multi_context=True)
-
-    mod.set_states(states=out)
-    mod.forward(batch)
-    out2 = mod.get_outputs(merge_multi_context=True)
-
-    for x1, x2 in zip(out1, out2):
-        assert not mx.test_utils.almost_equal(x1.asnumpy(), x2.asnumpy(), rtol=1e-3)
-
-
-@with_seed()
-def test_module_switch_bucket():
-    vocab_dim = 5000
-    num_hidden = 100
-    num_embedding = 100
-    num_layer = 2
-    default_key = 10
-    test_key = 5
-    batch_size = 32
-    contexts = [mx.cpu(0)]
-    initializer = mx.init.Xavier(factor_type="in", magnitude=2.34)
-
-    #generate symbols for an LSTM network
-    def sym_gen(seq_len):
-        data = mx.sym.Variable('data')
-        label = mx.sym.Variable('softmax_label')
-        embed = mx.sym.Embedding(data=data, input_dim=vocab_dim,
-                                 output_dim=num_embedding)
-        stack = mx.rnn.SequentialRNNCell()
-        for i in range(num_layer):
-            stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_'%i))
-        outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
-        pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
-        pred = mx.sym.FullyConnected(data=pred, num_hidden=vocab_dim, name='pred')
-
-        label = mx.sym.Reshape(label, shape=(-1,))
-        pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-
-        return pred, ('data',), ('softmax_label',)
-
-    def create_bucketing_module(key):
-        model = mx.mod.BucketingModule(
-            sym_gen             = sym_gen,
-            default_bucket_key  = key,
-            context             = contexts)
-        model.bind([('data', (batch_size, key))],
-                    [('softmax_label', (batch_size, key))], True, False)
-        model.init_params(initializer=initializer)
-        return model
-    #initialize the bucketing module with the default bucket key
-    bucketing_model = create_bucketing_module(default_key)
-    #check name
-    assert bucketing_model.symbol.list_arguments()[1] == "embedding0_weight",\
-        "Error in assigning names for args in BucketingModule"
-
-    #switch to test_key
-    bucketing_model.switch_bucket(test_key, [('data', (batch_size, test_key))],
-                                  [('softmax_label', (batch_size, test_key))])
-    total_bytes_before = bucketing_model._buckets[default_key]._total_exec_bytes
-
-    #remove test_key and switch again
-    del bucketing_model._buckets[test_key]
-    bucketing_model.switch_bucket(test_key, [('data', (batch_size, test_key))],
-                                  [('softmax_label', (batch_size, test_key))])
-    total_bytes_after = bucketing_model._buckets[default_key]._total_exec_bytes
-    #the default bucket is expected to reuse the bytes allocated
-    assert total_bytes_after == total_bytes_before
-
 
 # roywei: Getting rid of fixed seed as flakiness could not be reproduced,
 # tracked at: https://github.com/apache/incubator-mxnet/issues/11705
@@ -510,134 +352,6 @@ def test_monitor():
                 break
     assert(mon_result_counts == [2, 2, 1, 6, 6, 4])
 
-@with_seed()
-def test_executor_group():
-    def get_rnn_sym(num_layers, num_words, num_hidden, num_embed, seq_len, sparse_embedding):
-        stack = mx.rnn.SequentialRNNCell()
-        for i in range(num_layers):
-            stack.add(mx.rnn.LSTMCell(num_hidden=num_hidden, prefix='lstm_l%d_' % i))
-        data = mx.sym.Variable('data')
-        label = mx.sym.Variable('softmax_label')
-        if sparse_embedding:
-            embed_weight = mx.sym.Variable('embed_weight', stype='row_sparse')
-            embed = mx.sym.contrib.SparseEmbedding(data=data, input_dim=num_words,
-                                                   weight=embed_weight, output_dim=num_embed,
-                                                   name='embed')
-        else:
-            embed = mx.sym.Embedding(data=data, input_dim=num_words,
-                                     output_dim=num_embed, name='embed')
-
-        stack.reset()
-        outputs, states = stack.unroll(seq_len, inputs=embed, merge_outputs=True)
-
-        pred = mx.sym.Reshape(outputs, shape=(-1, num_hidden))
-        pred = mx.sym.FullyConnected(data=pred, num_hidden=num_words, name='pred')
-
-        label = mx.sym.Reshape(label, shape=(-1,))
-        pred = mx.sym.SoftmaxOutput(data=pred, label=label, name='softmax')
-        return pred
-
-    def test_shared_exec_group(exec_grp_shared, exec_grp_created, shared_arg_names=None,
-                               extra_args=None, check_shared_grad=True):
-        # Test shared data arrays
-        for i in range(len(exec_grp_shared.execs)):
-            # test same shared_data_arrays for two exec groups
-            shared_data_array1 = exec_grp_shared.shared_data_arrays[i]
-            shared_data_array2 = exec_grp_created.shared_data_arrays[i]
-            if extra_args is not None:
-                assert len(shared_data_array1) == len(extra_args),\
-                    "exec_grp_shared.shared_data_arrays[%d] should have same number of args as extra_args"
-            assert len(shared_data_array1) == len(shared_data_array2),\
-                "length of shared_data_array of the shared executor group not equal to the created executor group"
-            for k, v in shared_data_array1.items():
-                if extra_args is not None:
-                    assert k in extra_args, "arg %s is not in extra_args" % k
-                assert k in shared_data_array2,\
-                    "arg %s of the shared executor group not in the shared_data_array of the created executor group" % k
-                assert mx.test_utils.same_array(v, shared_data_array2[k])
-
-            for data_name, array in exec_grp_shared.shared_data_arrays[i].items():
-                assert data_name in exec_grp_created.shared_data_arrays[i], \
-                    "Shared input data '%s' is not in " \
-                    "shared_data_arrays of created executor group." % (data_name)
-                assert mx.test_utils.same_array(array, exec_grp_created.shared_data_arrays[i][data_name]), \
-                    "Shared input data '%s' does not share memory." % (data_name)
-
-            # Test shared argument arrays and gradient arrays
-            exec_shared = exec_grp_shared.execs[i]
-            exec_created = exec_grp_created.execs[i]
-            if shared_arg_names is not None:
-                # test shared arguments
-                for arg_name in shared_arg_names:
-                    assert arg_name in exec_created.arg_dict, \
-                        "Shared argument '%s' is not in arg_dict of created executor group." % (arg_name)
-                    assert mx.test_utils.same_array(exec_shared.arg_dict[arg_name], exec_created.arg_dict[arg_name]), \
-                        "Shared argument '%s' does not share memory." % (arg_name)
-                # test shared argument gradients
-                if check_shared_grad:
-                    for arg_name in shared_arg_names:
-                        assert arg_name in exec_created.grad_dict, \
-                            "Shared argument gradient '%s' is not in " \
-                            "grad_dict of created executor group." % (arg_name)
-                        assert mx.test_utils.same_array(exec_shared.grad_dict[arg_name], \
-                                                        exec_created.grad_dict[arg_name]), \
-                            "Shared argument gradient '%s' does not share memory." % (arg_name)
-
-            for arg_name, grad in exec_grp_shared.grad_req.items():
-                assert grad == exec_grp_created.grad_req[arg_name], \
-                    "Gradient requirements for shared argument '%s' are inconsistent. " \
-                    "Shared executor group requires '%s' while created executor group requires '%s'" \
-                    %(arg_name, grad, exec_grp_created.grad_req[arg_name])
-
-    def check_shared_exec_group(sparse_embedding):
-        # generate an rnn sym with #layers=5
-        sym = get_rnn_sym(num_layers=3, num_words=num_words, num_hidden=num_hidden,
-                          num_embed=num_embed, seq_len=max_bucket_size,
-                          sparse_embedding=sparse_embedding)
-        arg_names1 = sym.list_arguments()
-        input_names = [name[0] for name in data_shapes] + [name[0] for name in label_shapes]
-        shared_arg_names = [name for name in arg_names1 if name not in input_names]
-        exec_group1 = DataParallelExecutorGroup(symbol=sym, contexts=contexts,
-                                                workload=workload, data_shapes=data_shapes,
-                                                label_shapes=label_shapes, param_names=shared_arg_names,
-                                                for_training=True, inputs_need_grad=False)
-
-        # shared_data_arrays should only have input "data" and "softmax_label" arrays
-        for i in range(len(contexts)):
-            assert len(exec_group1.shared_data_arrays[i]) == len(input_names),\
-                "exec_group1.shared_data_arrays[%d] should have the same number of names as in input_names" % i
-            for name in input_names:
-                assert name in exec_group1.shared_data_arrays[i],\
-                    "arg %s should be in exec_group1.shared_data_arrays[%d]" % (name, i)
-
-        # generate an rnn sym with #layers=5
-        sym = get_rnn_sym(num_layers=5, num_words=num_words, num_hidden=num_hidden,
-                          num_embed=num_embed, seq_len=max_bucket_size,
-                          sparse_embedding=sparse_embedding)
-        arg_names2 = sym.list_arguments()
-        exec_group2 = DataParallelExecutorGroup(symbol=sym, contexts=contexts,
-                                                workload=workload, data_shapes=data_shapes,
-                                                label_shapes=label_shapes, param_names=shared_arg_names,
-                                                for_training=True, inputs_need_grad=False,
-                                                shared_group=exec_group1)
-        extra_args = [name for name in arg_names2 if name not in shared_arg_names]
-        check_shared_grad = not sparse_embedding
-        test_shared_exec_group(exec_grp_shared=exec_group1, exec_grp_created=exec_group2,
-                               shared_arg_names=shared_arg_names, extra_args=extra_args,
-                               check_shared_grad=check_shared_grad)
-
-    contexts = [mx.cpu(0), mx.cpu(1)]
-    workload = [1] * len(contexts)
-    batch_size = 32
-    max_bucket_size = 80
-    num_words = 1000
-    num_hidden = 100
-    num_embed = 200
-    data_shapes = [('data', (batch_size, max_bucket_size))]
-    label_shapes = [('softmax_label', (batch_size, max_bucket_size))]
-    sparse_embedding_opt = [True, False]
-    for opt in sparse_embedding_opt:
-        check_shared_exec_group(opt)
 
 @with_seed()
 def test_factorization_machine_module():
diff --git a/tests/python/unittest/test_operator.py b/tests/python/unittest/test_operator.py
index bdddf62..3a34f40 100644
--- a/tests/python/unittest/test_operator.py
+++ b/tests/python/unittest/test_operator.py
@@ -116,185 +116,6 @@ def test_rnn_with_new_param():
 
 
 @with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_lstm_sym():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='lstm', get_next_state=True, prefix='')
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.LSTMCell(H, prefix='l0_'))
-        stack.add(mx.rnn.LSTMCell(H, prefix='l1_'))
-        stack.add(mx.rnn.LSTMCell(H, prefix='l2_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_lstm_bidirectional():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='lstm',
-                                    bidirectional=True, get_next_state=True, prefix='')
-
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.LSTMCell(H, prefix='l0_'),
-                    mx.rnn.LSTMCell(H, prefix='r0_'),
-                    output_prefix='bi_lstm_0_'))
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.LSTMCell(H, prefix='l1_'),
-                    mx.rnn.LSTMCell(H, prefix='r1_'),
-                    output_prefix='bi_lstm_1_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-        check_rnn_consistency(fused, stack, T, N, I, H, {'data': 'add', 'parameters': 'null'})
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_gru_sym():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='gru', get_next_state=True, prefix='')
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.GRUCell(H, prefix='l0_'))
-        stack.add(mx.rnn.GRUCell(H, prefix='l1_'))
-        stack.add(mx.rnn.GRUCell(H, prefix='l2_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_gru_bidirectional():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='gru',
-                                    bidirectional=True, get_next_state=True, prefix='')
-
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.GRUCell(H, prefix='l0_'),
-                    mx.rnn.GRUCell(H, prefix='r0_'),
-                    output_prefix='bi_gru_0_'))
-
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.GRUCell(H, prefix='l1_'),
-                    mx.rnn.GRUCell(H, prefix='r1_'),
-                    output_prefix='bi_gru_1_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnntanh_sym():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='rnn_tanh', get_next_state=True, prefix='')
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l0_'))
-        stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l1_'))
-        stack.add(mx.rnn.RNNCell(H, activation='tanh', prefix='l2_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnntanh_bidirectional():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='rnn_tanh',
-                                    bidirectional=True, get_next_state=True, prefix='')
-
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.RNNCell(H, activation='tanh', prefix='l0_'),
-                    mx.rnn.RNNCell(H, activation='tanh', prefix='r0_'),
-                    output_prefix='bi_rnntanh_0_'))
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.RNNCell(H, activation='tanh', prefix='l1_'),
-                    mx.rnn.RNNCell(H, activation='tanh', prefix='r1_'),
-                    output_prefix='bi_rnntanh_1_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnnrelu_sym():
-    T, N, I, H = 5, 32, 200, 200
-
-    fused = mx.rnn.FusedRNNCell(H, num_layers=3, mode='rnn_relu', get_next_state=True, prefix='')
-    stack = mx.rnn.SequentialRNNCell()
-    stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l0_'))
-    stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l1_'))
-    stack.add(mx.rnn.RNNCell(H, activation='relu', prefix='l2_'))
-
-    check_rnn_consistency(fused, stack, T, N, I, H, 'write')
-    check_rnn_consistency(fused, stack, T, N, I, H, 'add')
-    check_rnn_consistency(fused, stack, T, N, I, H, 'null')
-
-@with_seed()
-@assert_raises_cudnn_not_satisfied(min_version='5.1.10')
-@pytest.mark.serial
-def test_rnnrelu_bidirectional():
-    Ts = [1, 5]
-    Ns = [1, 32]
-    Is = [32, 128, 512]
-    Hs = [32, 128, 512]
-    for T, N, I, H in itertools.product(Ts, Ns, Is, Hs):
-        fused = mx.rnn.FusedRNNCell(H, num_layers=2, mode='rnn_relu',
-                                    bidirectional=True, get_next_state=True, prefix='')
-
-        stack = mx.rnn.SequentialRNNCell()
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.RNNCell(H, activation='relu', prefix='l0_'),
-                    mx.rnn.RNNCell(H, activation='relu', prefix='r0_'),
-                    output_prefix='bi_rnnrelu_0_'))
-        stack.add(mx.rnn.BidirectionalCell(
-                    mx.rnn.RNNCell(H, activation='relu', prefix='l1_'),
-                    mx.rnn.RNNCell(H, activation='relu', prefix='r1_'),
-                    output_prefix='bi_rnnrelu_1_'))
-
-        check_rnn_consistency(fused, stack, T, N, I, H, 'write', rtol=1e-2, atol=1e-2)
-        check_rnn_consistency(fused, stack, T, N, I, H, 'add', rtol=1e-2, atol=1e-2)
-        check_rnn_consistency(fused, stack, T, N, I, H, 'null', rtol=1e-2, atol=1e-2)
-
-@with_seed()
 @pytest.mark.serial
 def test_lstm_dropout():
     X = mx.sym.Variable('x')
diff --git a/tests/python/unittest/test_rnn.py b/tests/python/unittest/test_rnn.py
deleted file mode 100644
index ab333f9..0000000
--- a/tests/python/unittest/test_rnn.py
+++ /dev/null
@@ -1,311 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-import mxnet as mx
-import numpy as np
-from numpy.testing import assert_allclose
-
-
-def test_deprecated():
-    class RNNCell(mx.rnn.BaseRNNCell):
-        """Simple recurrent neural network cell
-
-        Parameters
-        ----------
-        num_hidden : int
-            number of units in output symbol
-        activation : str or Symbol, default 'tanh'
-            type of activation function
-        prefix : str, default 'rnn_'
-            prefix for name of layers
-            (and name of weight if params is None)
-        params : RNNParams or None
-            container for weight sharing between cells.
-            created if None.
-        """
-        def __init__(self, num_hidden, activation='tanh', prefix='rnn_', params=None):
-            super(RNNCell, self).__init__(prefix=prefix, params=params)
-            self._num_hidden = num_hidden
-            self._activation = activation
-            self._iW = self.params.get('i2h_weight')
-            self._iB = self.params.get('i2h_bias')
-            self._hW = self.params.get('h2h_weight')
-            self._hB = self.params.get('h2h_bias')
-
-        @property
-        def state_info(self):
-            return [{'shape': (0, self._num_hidden), '__layout__': 'NC'}]
-
-        @property
-        def _gate_names(self):
-            return ('',)
-
-        def __call__(self, inputs, states):
-            self._counter += 1
-            name = '%st%d_'%(self._prefix, self._counter)
-            i2h = mx.symbol.FullyConnected(data=inputs, weight=self._iW, bias=self._iB,
-                                        num_hidden=self._num_hidden,
-                                        name='%si2h'%name)
-            h2h = mx.symbol.FullyConnected(data=states[0], weight=self._hW, bias=self._hB,
-                                        num_hidden=self._num_hidden,
-                                        name='%sh2h'%name)
-            output = self._get_activation(i2h + h2h, self._activation,
-                                          name='%sout'%name)
-
-            return output, [output]
-
-    cell = RNNCell(100, prefix='rnn_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_rnn():
-    cell = mx.rnn.RNNCell(100, prefix='rnn_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_lstm():
-    for activation_type in ['', 'relu', 'sigmoid', 'softrelu', 'tanh', 'softsign']:
-        if activation_type == '':
-            cell = mx.gluon.rnn.LSTMCell(100, prefix='rnn_')
-        else:
-            cell = mx.gluon.rnn.LSTMCell(100, prefix='rnn_', activation=activation_type, recurrent_activation=activation_type)
-        inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-        outputs, _ = cell.unroll(3, inputs)
-        outputs = mx.sym.Group(outputs)
-        assert sorted(cell.collect_params().keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-        assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-        args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-        assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_lstm_forget_bias():
-    forget_bias = 2.0
-    stack = mx.rnn.SequentialRNNCell()
-    stack.add(mx.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l0_'))
-    stack.add(mx.rnn.LSTMCell(100, forget_bias=forget_bias, prefix='l1_'))
-
-    dshape = (32, 1, 200)
-    data = mx.sym.Variable('data')
-
-    sym, _ = stack.unroll(1, data, merge_outputs=True)
-    mod = mx.mod.Module(sym, label_names=None, context=mx.cpu(0))
-    mod.bind(data_shapes=[('data', dshape)], label_shapes=None)
-
-    mod.init_params()
-
-    bias_argument = next(x for x in sym.list_arguments() if x.endswith('i2h_bias'))
-    expected_bias = np.hstack([np.zeros((100,)),
-                               forget_bias * np.ones(100, ), np.zeros((2 * 100,))])
-    assert_allclose(mod.get_params()[0][bias_argument].asnumpy(), expected_bias)
-
-
-def test_gru():
-    cell = mx.rnn.GRUCell(100, prefix='rnn_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_residual():
-    cell = mx.rnn.ResidualCell(mx.rnn.GRUCell(50, prefix='rnn_'))
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)]
-    outputs, _ = cell.unroll(2, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == \
-           ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    # assert outputs.list_outputs() == \
-    #        ['rnn_t0_out_plus_residual_output', 'rnn_t1_out_plus_residual_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50))
-    assert outs == [(10, 50), (10, 50)]
-    outputs = outputs.eval(rnn_t0_data=mx.nd.ones((10, 50)),
-                           rnn_t1_data=mx.nd.ones((10, 50)),
-                           rnn_i2h_weight=mx.nd.zeros((150, 50)),
-                           rnn_i2h_bias=mx.nd.zeros((150,)),
-                           rnn_h2h_weight=mx.nd.zeros((150, 50)),
-                           rnn_h2h_bias=mx.nd.zeros((150,)))
-    expected_outputs = np.ones((10, 50))
-    assert np.array_equal(outputs[0].asnumpy(), expected_outputs)
-    assert np.array_equal(outputs[1].asnumpy(), expected_outputs)
-
-
-def test_residual_bidirectional():
-    cell = mx.rnn.ResidualCell(
-            mx.rnn.BidirectionalCell(
-                mx.rnn.GRUCell(25, prefix='rnn_l_'),
-                mx.rnn.GRUCell(25, prefix='rnn_r_')))
-
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(2)]
-    outputs, _ = cell.unroll(2, inputs, merge_outputs=False)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == \
-           ['rnn_l_h2h_bias', 'rnn_l_h2h_weight', 'rnn_l_i2h_bias', 'rnn_l_i2h_weight',
-            'rnn_r_h2h_bias', 'rnn_r_h2h_weight', 'rnn_r_i2h_bias', 'rnn_r_i2h_weight']
-    # assert outputs.list_outputs() == \
-    #        ['bi_t0_plus_residual_output', 'bi_t1_plus_residual_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10, 50), rnn_t1_data=(10, 50))
-    assert outs == [(10, 50), (10, 50)]
-    outputs = outputs.eval(rnn_t0_data=mx.nd.ones((10, 50))+5,
-                           rnn_t1_data=mx.nd.ones((10, 50))+5,
-                           rnn_l_i2h_weight=mx.nd.zeros((75, 50)),
-                           rnn_l_i2h_bias=mx.nd.zeros((75,)),
-                           rnn_l_h2h_weight=mx.nd.zeros((75, 25)),
-                           rnn_l_h2h_bias=mx.nd.zeros((75,)),
-                           rnn_r_i2h_weight=mx.nd.zeros((75, 50)),
-                           rnn_r_i2h_bias=mx.nd.zeros((75,)),
-                           rnn_r_h2h_weight=mx.nd.zeros((75, 25)),
-                           rnn_r_h2h_bias=mx.nd.zeros((75,)))
-    expected_outputs = np.ones((10, 50))+5
-    assert np.array_equal(outputs[0].asnumpy(), expected_outputs)
-    assert np.array_equal(outputs[1].asnumpy(), expected_outputs)
-
-
-def test_stack():
-    cell = mx.rnn.SequentialRNNCell()
-    for i in range(5):
-        if i == 1:
-            cell.add(mx.rnn.ResidualCell(mx.rnn.LSTMCell(100, prefix='rnn_stack%d_' % i)))
-        else:
-            cell.add(mx.rnn.LSTMCell(100, prefix='rnn_stack%d_'%i))
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    keys = sorted(cell.params._params.keys())
-    for i in range(5):
-        assert 'rnn_stack%d_h2h_weight'%i in keys
-        assert 'rnn_stack%d_h2h_bias'%i in keys
-        assert 'rnn_stack%d_i2h_weight'%i in keys
-        assert 'rnn_stack%d_i2h_bias'%i in keys
-    assert outputs.list_outputs() == ['rnn_stack4_t0_out_output', 'rnn_stack4_t1_out_output', 'rnn_stack4_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_bidirectional():
-    cell = mx.rnn.BidirectionalCell(
-            mx.rnn.LSTMCell(100, prefix='rnn_l0_'),
-            mx.rnn.LSTMCell(100, prefix='rnn_r0_'),
-            output_prefix='rnn_bi_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert outputs.list_outputs() == ['rnn_bi_t0_output', 'rnn_bi_t1_output', 'rnn_bi_t2_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 200), (10, 200), (10, 200)]
-
-
-def test_zoneout():
-    cell = mx.rnn.ZoneoutCell(mx.rnn.RNNCell(100, prefix='rnn_'), zoneout_outputs=0.5,
-                              zoneout_states=0.5)
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 100), (10, 100), (10, 100)]
-
-
-def test_unfuse():
-    cell = mx.rnn.FusedRNNCell(100, num_layers=3, mode='lstm',
-                               prefix='test_', bidirectional=True,
-                               dropout=0.5)
-    cell = cell.unfuse()
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert outputs.list_outputs() == ['test_bi_l2_t0_output', 'test_bi_l2_t1_output', 'test_bi_l2_t2_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(10,50), rnn_t1_data=(10,50), rnn_t2_data=(10,50))
-    assert outs == [(10, 200), (10, 200), (10, 200)]
-
-def test_convrnn():
-    cell = mx.rnn.ConvRNNCell(input_shape = (1, 3, 16, 10), num_hidden=10,
-                              h2h_kernel=(3, 3), h2h_dilate=(1, 1),
-                              i2h_kernel=(3, 3), i2h_stride=(1, 1),
-                              i2h_pad=(1, 1), i2h_dilate=(1, 1),
-                              prefix='rnn_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10))
-    assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)]
-
-def test_convlstm():
-    cell = mx.rnn.ConvLSTMCell(input_shape = (1, 3, 16, 10), num_hidden=10,
-                               h2h_kernel=(3, 3), h2h_dilate=(1, 1),
-                               i2h_kernel=(3, 3), i2h_stride=(1, 1),
-                               i2h_pad=(1, 1), i2h_dilate=(1, 1),
-                               prefix='rnn_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10))
-    assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)]
-
-def test_convgru():
-    cell = mx.rnn.ConvGRUCell(input_shape = (1, 3, 16, 10), num_hidden=10,
-                              h2h_kernel=(3, 3), h2h_dilate=(1, 1),
-                              i2h_kernel=(3, 3), i2h_stride=(1, 1),
-                              i2h_pad=(1, 1), i2h_dilate=(1, 1),
-                              prefix='rnn_')
-    inputs = [mx.sym.Variable('rnn_t%d_data'%i) for i in range(3)]
-    outputs, _ = cell.unroll(3, inputs)
-    outputs = mx.sym.Group(outputs)
-    assert sorted(cell.params._params.keys()) == ['rnn_h2h_bias', 'rnn_h2h_weight', 'rnn_i2h_bias', 'rnn_i2h_weight']
-    assert outputs.list_outputs() == ['rnn_t0_out_output', 'rnn_t1_out_output', 'rnn_t2_out_output']
-
-    args, outs, auxs = outputs.infer_shape(rnn_t0_data=(1, 3, 16, 10), rnn_t1_data=(1, 3, 16, 10), rnn_t2_data=(1, 3, 16, 10))
-    assert outs == [(1, 10, 16, 10), (1, 10, 16, 10), (1, 10, 16, 10)]
-
-def test_encode_sentences():
-    sentences = [['a','b','c'],['b','c','d']]
-    dict = {'a':1, 'b':2, 'c':3}
-    result, vocab = mx.rnn.io.encode_sentences(sentences, vocab=dict, invalid_label=-1, invalid_key='\n',
-                         start_label=0, unknown_token='UNK')
-    print(result, vocab)
-    assert vocab == {'a': 1, 'b': 2, 'c': 3, 'UNK': 0}
-    assert result == [[1,2,3],[2,3,0]]
-