You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2018/10/29 07:16:55 UTC
[incubator-mxnet] branch master updated: [MXNET -1030] Cosine
Embedding Loss (#12750)
This is an automated email from the ASF dual-hosted git repository.
zhasheng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 1f971c2 [MXNET -1030] Cosine Embedding Loss (#12750)
1f971c2 is described below
commit 1f971c2b0ad0e8931126e26420b6aeb8382f68e4
Author: Gaurav Gireesh <Ga...@fox.com>
AuthorDate: Mon Oct 29 00:16:42 2018 -0700
[MXNET -1030] Cosine Embedding Loss (#12750)
* COsine Embedding Loss function added
* Added unit tests for Cosine Embedding Loss Function
* Added Latex code for formula for cosine embedding loss
* Fixing document rendering
* Fixing documentation issue
* PR Comments addressed for using F (NDArray or Symbol) to calculate norm, renaming parameters
* Markdown file updated. Added entry for CosineEmbeddingLoss
* Added a line after .. math:: to fix documentation
* Documentation check - pylint fix
* Formula update
* Making the formula simpler for correct rendering incrementally - Update 1
* Making the formula simpler for correct rendering incrementally - Update 2
* Making the formula simpler for correct rendering incrementally - Update 3
* Making the formula simpler for correct rendering incrementally - Update 4
* Making the formula simpler for correct rendering incrementally - Update 5
* Trigger CI
* making the utility function cosine similarity internal
* Added a test case for label = -1, for dissimilar vectors
* Refactored names of parameters to the loss functions and updated the formula in docstring
* PR comments addressed changes in documentation
* Added random input vectors and labelled tests
* Renaming variables
* Pylint issues fixed
* Resolving conflicts
* Pylint issues fixed
* Style issues fixed trailing whitespaces removed
* Review comment addressed, sample_weight added in the parameter
* Trigger CI
* Reordered Parameter description
* comments addressed - spelling errors
* nit comments addressed
* Trigger CI
* Trugger CI
* Trigger CI
* Trigger CI
---
docs/api/python/gluon/loss.md | 1 +
python/mxnet/gluon/loss.py | 70 +++++++++++++++++++++++++++++++++++++-
tests/python/unittest/test_loss.py | 18 ++++++++++
3 files changed, 88 insertions(+), 1 deletion(-)
diff --git a/docs/api/python/gluon/loss.md b/docs/api/python/gluon/loss.md
index 3747a0f..948f498 100644
--- a/docs/api/python/gluon/loss.md
+++ b/docs/api/python/gluon/loss.md
@@ -25,6 +25,7 @@ This package includes several commonly used loss functions in neural networks.
LogisticLoss
TripletLoss
CTCLoss
+ CosineEmbeddingLoss
PoissonNLLLoss
```
diff --git a/python/mxnet/gluon/loss.py b/python/mxnet/gluon/loss.py
index 5d3ebb7..da43b62 100644
--- a/python/mxnet/gluon/loss.py
+++ b/python/mxnet/gluon/loss.py
@@ -23,7 +23,7 @@ __all__ = ['Loss', 'L2Loss', 'L1Loss',
'SigmoidBinaryCrossEntropyLoss', 'SigmoidBCELoss',
'SoftmaxCrossEntropyLoss', 'SoftmaxCELoss',
'KLDivLoss', 'CTCLoss', 'HuberLoss', 'HingeLoss',
- 'SquaredHingeLoss', 'LogisticLoss', 'TripletLoss', 'PoissonNLLLoss']
+ 'SquaredHingeLoss', 'LogisticLoss', 'TripletLoss', 'PoissonNLLLoss', 'CosineEmbeddingLoss']
import numpy as np
from .. import ndarray
@@ -767,3 +767,71 @@ class PoissonNLLLoss(Loss):
loss += stirling_factor
loss = _apply_weighting(F, loss, self._weight, sample_weight)
return F.mean(loss)
+
+
+class CosineEmbeddingLoss(Loss):
+ r"""For a target label 1 or -1, vectors input1 and input2, the function computes the cosine distance
+ between the vectors. This can be interpreted as how similar/dissimilar two input vectors are.
+
+ .. math::
+
+ L = \sum_i \begin{cases} 1 - {cos\_sim({input1}_i, {input2}_i)} & \text{ if } {label}_i = 1\\
+ {cos\_sim({input1}_i, {input2}_i)} & \text{ if } {label}_i = -1 \end{cases}\\
+ cos\_sim(input1, input2) = \frac{{input1}_i.{input2}_i}{||{input1}_i||.||{input2}_i||}
+
+ `input1`, `input2` can have arbitrary shape as long as they have the same number of elements.
+
+ Parameters
+ ----------
+ weight : float or None
+ Global scalar weight for loss.
+ batch_axis : int, default 0
+ The axis that represents mini-batch.
+ margin : float
+ Margin of separation between correct and incorrect pair.
+
+
+ Inputs:
+ - **input1**: a tensor with arbitrary shape
+ - **input2**: another tensor with same shape as pred to which input1 is
+ compared for similarity and loss calculation
+ - **label**: A 1-D tensor indicating for each pair input1 and input2, target label is 1 or -1
+ - **sample_weight**: element-wise weighting tensor. Must be broadcastable
+ to the same shape as input1. For example, if input1 has shape (64, 10)
+ and you want to weigh each sample in the batch separately,
+ sample_weight should have shape (64, 1).
+
+ Outputs:
+ - **loss**: The loss tensor with shape (batch_size,).
+ """
+ def __init__(self, weight=None, batch_axis=0, margin=0, **kwargs):
+ super(CosineEmbeddingLoss, self).__init__(weight, batch_axis, **kwargs)
+ self._margin = margin
+
+ def hybrid_forward(self, F, input1, input2, label, sample_weight=None):
+ input1 = _reshape_like(F, input1, input2)
+ label = label.reshape((-1, 1))
+ cos_sim = self._cosine_similarity(F, input1, input2)
+ y_1 = label == 1
+ y_minus_1 = label == -1
+ cos_sim_a = (1 - cos_sim) * y_1
+
+ if F is ndarray:
+ z_array = F.array([0])
+ else:
+ z_array = F.zeros((1, 1))
+ cos_sim_b = F.broadcast_maximum(z_array, y_minus_1 * (cos_sim - self._margin), axis=1)
+ loss = cos_sim_a + cos_sim_b
+ loss = _apply_weighting(F, loss, self._weight, sample_weight)
+ return loss
+
+ def _cosine_similarity(self, F, x, y, axis=-1):
+ # Calculates the cosine similarity between 2 vectors
+ x_norm = F.norm(x, axis=axis).reshape(-1, 1)
+ y_norm = F.norm(y, axis=axis).reshape(-1, 1)
+ x_dot_y = F.sum(x*y, axis=axis).reshape(-1, 1)
+ if F is ndarray:
+ eps_arr = F.array([1e-12])
+ else:
+ eps_arr = F.full((1, 1), 1e-12)
+ return (x_dot_y / F.broadcast_maximum(x_norm * y_norm, eps_arr))
diff --git a/tests/python/unittest/test_loss.py b/tests/python/unittest/test_loss.py
index 2b062fb..18d1ebf 100644
--- a/tests/python/unittest/test_loss.py
+++ b/tests/python/unittest/test_loss.py
@@ -349,6 +349,23 @@ def test_triplet_loss():
assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
@with_seed()
+def test_cosine_loss():
+ #Generating samples
+ input1 = mx.nd.random.randn(3, 2)
+ input2 = mx.nd.random.randn(3, 2)
+ label = mx.nd.sign(mx.nd.random.randn(input1.shape[0]))
+ #Calculating loss from cosine embedding loss function in Gluon
+ Loss = gluon.loss.CosineEmbeddingLoss()
+ loss = Loss(input1, input2, label)
+
+ # Calculating the loss Numpy way
+ numerator = mx.nd.sum(input1 * input2, keepdims=True, axis=1)
+ denominator = mx.nd.sqrt(mx.nd.sum(input1**2, axis=1, keepdims=True)) \
+ * mx.nd.sqrt(mx.nd.sum(input2**2, axis=1, keepdims=True))
+ numpy_loss = mx.nd.where(label == 1, 1-numerator/denominator, \
+ mx.nd.broadcast_maximum(mx.nd.array([0]), numerator/denominator, axis=1))
+ assert_almost_equal(loss.asnumpy(), numpy_loss.asnumpy(), rtol=1e-3, atol=1e-5)
+
def test_poisson_nllloss():
pred = mx.nd.random.normal(shape=(3, 4))
min_pred = mx.nd.min(pred)
@@ -404,6 +421,7 @@ def test_poisson_nllloss_mod():
optimizer='adam')
assert mod.score(data_iter, eval_metric=mx.metric.Loss())[0][1] < 0.05
+
if __name__ == '__main__':
import nose
nose.runmodule()