You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@labs.apache.org by to...@apache.org on 2015/12/17 08:04:13 UTC
svn commit: r1720488 - in /labs/yay/trunk: api/src/main/java/org/apache/yay/
core/src/main/java/org/apache/yay/core/
core/src/main/java/org/apache/yay/core/utils/
core/src/test/java/org/apache/yay/core/ core/src/test/resources/word2vec/
Author: tommaso
Date: Thu Dec 17 07:04:12 2015
New Revision: 1720488
URL: http://svn.apache.org/viewvc?rev=1720488&view=rev
Log:
performance improvements (parallel exec, simplified Feature APIs), revert derivative update function to latest working version
Removed:
labs/yay/trunk/api/src/main/java/org/apache/yay/Feature.java
Modified:
labs/yay/trunk/api/src/main/java/org/apache/yay/Input.java
labs/yay/trunk/api/src/main/java/org/apache/yay/PredictionStrategy.java
labs/yay/trunk/api/src/main/java/org/apache/yay/WeightLearningException.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/BackPropagationLearningStrategy.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/BasicPerceptron.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/DefaultDerivativeUpdateFunction.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/EncodedTrainingSet.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/FeedForwardStrategy.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/LogisticRegressionCostFunction.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/MaxSelectionFunction.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/NeuralNetworkFactory.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ConversionUtils.java
labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ExamplesFactory.java
labs/yay/trunk/core/src/test/java/org/apache/yay/core/BasicPerceptronTest.java
labs/yay/trunk/core/src/test/java/org/apache/yay/core/NeuralNetworkIntegrationTest.java
labs/yay/trunk/core/src/test/java/org/apache/yay/core/WordVectorsTest.java
labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt
Modified: labs/yay/trunk/api/src/main/java/org/apache/yay/Input.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/api/src/main/java/org/apache/yay/Input.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/api/src/main/java/org/apache/yay/Input.java (original)
+++ labs/yay/trunk/api/src/main/java/org/apache/yay/Input.java Thu Dec 17 07:04:12 2015
@@ -18,7 +18,7 @@
*/
package org.apache.yay;
-import java.util.ArrayList;
+import java.util.List;
/**
* A sample/input characterized by its features.
@@ -26,10 +26,10 @@ import java.util.ArrayList;
public interface Input<F> {
/**
- * Get this <code>Input</code> {@link org.apache.yay.Feature}s
+ * Get this <code>Input</code> features
*
- * @return an <code>ArrayList</code> of {@link org.apache.yay.Feature}s
+ * @return an <code>ArrayList</code> of features
*/
- ArrayList<Feature<F>> getFeatures();
+ List<F> getFeatures();
}
Modified: labs/yay/trunk/api/src/main/java/org/apache/yay/PredictionStrategy.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/api/src/main/java/org/apache/yay/PredictionStrategy.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/api/src/main/java/org/apache/yay/PredictionStrategy.java (original)
+++ labs/yay/trunk/api/src/main/java/org/apache/yay/PredictionStrategy.java Thu Dec 17 07:04:12 2015
@@ -18,10 +18,11 @@
*/
package org.apache.yay;
-import java.util.Collection;
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.commons.math3.linear.RealVector;
+import java.util.Collection;
+
/**
* A {@link PredictionStrategy} defines an algorithm for the prediction of outputs
* of type <code>O</code> given inputs of type <code>I</code>.
@@ -38,6 +39,15 @@ public interface PredictionStrategy<I, O
O[] predictOutput(Collection<I> inputs, RealMatrix[] weightsMatrixSet);
/**
+ * Perform a prediction and returns a vector containing the outputs
+ *
+ * @param inputVector a vector of input values
+ * @param weightsMatrixSet the initial set of weights defined by an array of matrix
+ * @return the array containing the last layer's outputs
+ */
+ RealVector predictOutput(RealVector inputVector, RealMatrix[] weightsMatrixSet);
+
+ /**
* Perform a prediction on the given input values and weights settings returning
* a debug output.
*
@@ -47,4 +57,14 @@ public interface PredictionStrategy<I, O
*/
RealVector[] debugOutput(Collection<I> inputs, RealMatrix[] weightsMatrixSet);
+ /**
+ * Perform a prediction on the given input values and weights settings returning
+ * a debug output.
+ *
+ * @param inputVector a vector of input values
+ * @param weightsMatrixSet the initial set of weights defined by an array of matrix
+ * @return the perturbed neural network state via its activations values
+ */
+ RealVector[] debugOutput(RealVector inputVector, RealMatrix[] weightsMatrixSet);
+
}
Modified: labs/yay/trunk/api/src/main/java/org/apache/yay/WeightLearningException.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/api/src/main/java/org/apache/yay/WeightLearningException.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/api/src/main/java/org/apache/yay/WeightLearningException.java (original)
+++ labs/yay/trunk/api/src/main/java/org/apache/yay/WeightLearningException.java Thu Dec 17 07:04:12 2015
@@ -30,4 +30,8 @@ public class WeightLearningException ext
public WeightLearningException(String s, Exception e) {
super(s, e);
}
+
+ public WeightLearningException(String s) {
+ super(s);
+ }
}
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/BackPropagationLearningStrategy.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/BackPropagationLearningStrategy.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/BackPropagationLearningStrategy.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/BackPropagationLearningStrategy.java Thu Dec 17 07:04:12 2015
@@ -25,12 +25,14 @@ import org.apache.yay.DerivativeUpdateFu
import org.apache.yay.LearningStrategy;
import org.apache.yay.NeuralNetwork;
import org.apache.yay.PredictionStrategy;
-import org.apache.yay.TrainingExample;
import org.apache.yay.TrainingSet;
import org.apache.yay.WeightLearningException;
-import java.util.Arrays;
-import java.util.Iterator;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
/**
* Back propagation learning algorithm for neural networks implementation (see
@@ -49,6 +51,7 @@ public class BackPropagationLearningStra
private final double threshold;
private final int batch;
private final int maxIterations;
+ private final ExecutorService executorService = Executors.newCachedThreadPool();
public BackPropagationLearningStrategy(double alpha, double threshold, PredictionStrategy<Double, Double> predictionStrategy,
CostFunction<RealMatrix, Double, Double> costFunction) {
@@ -79,12 +82,11 @@ public class BackPropagationLearningStra
@Override
public RealMatrix[] learnWeights(RealMatrix[] weightsMatrixSet, TrainingSet<Double, Double> trainingExamples) throws WeightLearningException {
- RealMatrix[] updatedWeights = weightsMatrixSet;
+ RealMatrix[] weights = weightsMatrixSet;
try {
int iterations = 0;
NeuralNetwork neuralNetwork = NeuralNetworkFactory.create(weightsMatrixSet, new VoidLearningStrategy<>(), predictionStrategy);
- Iterator<TrainingExample<Double, Double>> iterator = trainingExamples.iterator();
double cost = Double.MAX_VALUE;
long start = System.currentTimeMillis();
@@ -100,12 +102,12 @@ public class BackPropagationLearningStra
double newCost = costFunction.calculateCost(nextBatch, neuralNetwork);
if (Double.POSITIVE_INFINITY == newCost || newCost > cost && batch == -1) {
- throw new RuntimeException("failed to converge at iteration " + iterations + " with alpha " + alpha + " : cost going from " + cost + " to " + newCost);
+ throw new WeightLearningException("failed to converge at iteration " + iterations + " with alpha " + alpha + " : cost going from " + cost + " to " + newCost);
} else if (iterations > 1 && (cost == newCost || newCost < threshold || iterations > maxIterations)) {
- System.out.println("successfully converged after " + (iterations - 1) + " iterations (alpha:" + alpha + ",threshold:" + threshold + ") with cost " + newCost + " and parameters " + Arrays.toString(neuralNetwork.getParameters()));
+ System.out.println("successfully converged after " + (iterations - 1) + " iterations (alpha:" + alpha + ",threshold:" + threshold + ") with cost " + newCost);
break;
} else if (Double.isNaN(newCost)) {
- throw new RuntimeException("failed to converge at iteration " + iterations + " with alpha " + alpha + " : cost calculation underflow");
+ throw new WeightLearningException("failed to converge at iteration " + iterations + " with alpha " + alpha + " : cost calculation underflow");
}
// update registered cost
@@ -115,52 +117,72 @@ public class BackPropagationLearningStra
RealMatrix[] derivatives = derivativeUpdateFunction.getUpdatedDerivatives(weightsMatrixSet, nextBatch);
// calculate the updated parameters
- updatedWeights = updateWeights(updatedWeights, derivatives, alpha);
+ weights = getUpdatedWeights(weights, derivatives, alpha);
// update parameters in the hypothesis
- neuralNetwork.setParameters(updatedWeights);
+ neuralNetwork.setParameters(weights);
iterations++;
}
+ } catch (WeightLearningException e) {
+ throw e;
} catch (Exception e) {
throw new WeightLearningException("error during backprop learning", e);
}
- return updatedWeights;
+ return weights;
}
- private RealMatrix[] updateWeights(RealMatrix[] weightsMatrixSet, final RealMatrix[] derivatives, final double alpha) {
- RealMatrix[] updatedParameters = new RealMatrix[weightsMatrixSet.length];
+ private RealMatrix[] getUpdatedWeights(RealMatrix[] weightsMatrixSet, final RealMatrix[] derivatives, final double alpha) {
+ int length = weightsMatrixSet.length;
+
+ RealMatrix[] updatedParameters = new RealMatrix[length];
+
+ List<Future<RealMatrix>> futures = new ArrayList<>(length);
+
+ for (int l = 0; l < length; l++) {
- for (int l = 0; l < weightsMatrixSet.length; l++) {
RealMatrix realMatrix = weightsMatrixSet[l].copy();
final int finalL = l;
- RealMatrixChangingVisitor visitor = new RealMatrixChangingVisitor() {
+ futures.add(executorService.submit(() -> {
+ RealMatrixChangingVisitor visitor = new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int startColumn, int endColumn) {
+ @Override
+ public void start(int rows, int columns, int startRow, int endRow, int startColumn, int endColumn) {
- }
+ }
- @Override
- public double visit(int row, int column, double value) {
- if (!(row == 0 && value == 0d) && !(column == 0 && value == 1d)) {
- return value - alpha * derivatives[finalL].getEntry(row, column);
- } else {
- return value;
+ @Override
+ public double visit(int row, int column, double value) {
+ if (!(row == 0 && value == 0d) && !(column == 0 && value == 1d)) {
+ return value - alpha * derivatives[finalL].getEntry(row, column);
+ } else {
+ return value;
+ }
}
- }
- @Override
- public double end() {
- return 0;
+ @Override
+ public double end() {
+ return 0;
+ }
+ };
+ realMatrix.walkInOptimizedOrder(visitor);
+ return realMatrix;
+ }));
+
+ }
+ int k = 0;
+ for (Future<RealMatrix> future : futures) {
+ if (updatedParameters[k] == null) {
+ try {
+ updatedParameters[k] = future.get();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
}
- };
- realMatrix.walkInOptimizedOrder(visitor);
- if (updatedParameters[l] == null) {
- updatedParameters[l] = realMatrix;
}
+ k++;
}
+
return updatedParameters;
}
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/BasicPerceptron.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/BasicPerceptron.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/BasicPerceptron.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/BasicPerceptron.java Thu Dec 17 07:04:12 2015
@@ -27,7 +27,6 @@ import org.apache.yay.PredictionExceptio
import org.apache.yay.TrainingExample;
import org.apache.yay.TrainingSet;
import org.apache.yay.core.neuron.BinaryThresholdNeuron;
-import org.apache.yay.core.utils.ConversionUtils;
import java.util.Collection;
@@ -61,7 +60,7 @@ public class BasicPerceptron implements
}
public void learn(TrainingExample<Double, Double> example) {
- Collection<Double> doubles = ConversionUtils.toValuesCollection(example.getFeatures());
+ Collection<Double> doubles = example.getFeatures();
Double[] inputs = doubles.toArray(new Double[doubles.size()]);
Double calculatedOutput = perceptronNeuron.elaborate(inputs);
int diff = calculatedOutput.compareTo(example.getOutput()[0]);
@@ -93,7 +92,7 @@ public class BasicPerceptron implements
@Override
public Double[] predict(Input<Double> input) throws PredictionException {
- Double output = perceptronNeuron.elaborate(ConversionUtils.toValuesCollection(input.getFeatures()).toArray(
+ Double output = perceptronNeuron.elaborate(input.getFeatures().toArray(
new Double[input.getFeatures().size()]));
return new Double[]{output};
}
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/DefaultDerivativeUpdateFunction.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/DefaultDerivativeUpdateFunction.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/DefaultDerivativeUpdateFunction.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/DefaultDerivativeUpdateFunction.java Thu Dec 17 07:04:12 2015
@@ -18,20 +18,15 @@
*/
package org.apache.yay.core;
-import org.apache.commons.math3.linear.OpenMapRealVector;
+import org.apache.commons.math3.linear.ArrayRealVector;
import org.apache.commons.math3.linear.RealMatrix;
-import org.apache.commons.math3.linear.RealMatrixChangingVisitor;
import org.apache.commons.math3.linear.RealVector;
import org.apache.yay.DerivativeUpdateFunction;
-import org.apache.yay.Feature;
import org.apache.yay.PredictionStrategy;
import org.apache.yay.TrainingExample;
import org.apache.yay.TrainingSet;
import org.apache.yay.core.utils.ConversionUtils;
-import java.util.ArrayList;
-import java.util.Collection;
-
/**
* Default derivatives update function
*/
@@ -50,71 +45,49 @@ class DefaultDerivativeUpdateFunction im
RealVector[] deltaVectors = new RealVector[weightsMatrixSet.length];
int noOfMatrixes = weightsMatrixSet.length - 1;
+ double count = 0;
for (TrainingExample<Double, Double> trainingExample : trainingExamples) {
try {
// get activations from feed forward propagation
- ArrayList<Feature<Double>> features = trainingExample.getFeatures();
- Collection<Double> input = ConversionUtils.toValuesCollection(features);
-
- RealVector[] activations = predictionStrategy.debugOutput(input, weightsMatrixSet);
+ RealVector[] activations = predictionStrategy.debugOutput(trainingExample.getFeatures(), weightsMatrixSet);
// calculate output error (corresponding to the last delta^l)
RealVector nextLayerDelta = calculateOutputError(trainingExample, activations);
- updateDeltaVectors(weightsMatrixSet, deltaVectors, noOfMatrixes, activations, nextLayerDelta);
+ deltaVectors[noOfMatrixes] = nextLayerDelta;
+
+ // back prop the error and update the deltas accordingly
+ for (int l = noOfMatrixes; l > 0; l--) {
+ RealVector currentActivationsVector = activations[l - 1];
+ nextLayerDelta = calculateDeltaVector(weightsMatrixSet[l], currentActivationsVector, nextLayerDelta);
+
+ // collect delta vectors for this example
+ deltaVectors[l - 1] = nextLayerDelta;
+ }
RealVector[] newActivations = new RealVector[activations.length];
- newActivations[0] = ConversionUtils.toRealVector(input);
+ newActivations[0] = ConversionUtils.toRealVector(trainingExample.getFeatures());
System.arraycopy(activations, 0, newActivations, 1, activations.length - 1);
// update triangle (big delta matrix)
- updateTriangle(triangle, newActivations, deltaVectors);
+ updateTriangle(triangle, newActivations, deltaVectors, weightsMatrixSet);
+
} catch (Exception e) {
throw new RuntimeException("error during derivatives calculation", e);
}
+ count++;
}
- for (RealMatrix aTriangle : triangle) {
- aTriangle.walkInOptimizedOrder(new RealMatrixChangingVisitor() {
- @Override
- public void start(int rows, int columns, int startRow, int endRow, int startColumn, int endColumn) {
-
- }
-
- @Override
- public double visit(int row, int column, double value) {
- if (!(row == 0 && value == 0d) && !(column == 0 && value == 1d)) {
- return value / trainingExamples.size();
- } else {
- return value;
- }
- }
-
- @Override
- public double end() {
- return 0;
- }
- });
- }
- return triangle;
- }
-
- private void updateDeltaVectors(RealMatrix[] weightsMatrixSet, RealVector[] deltaVectors, int noOfMatrixes, RealVector[] activations,
- RealVector nextLayerDelta) {
- deltaVectors[noOfMatrixes] = nextLayerDelta;
-
- // back prop the error and update the deltas accordingly
- for (int l = noOfMatrixes; l > 0; l--) {
- RealVector currentActivationsVector = activations[l - 1];
- nextLayerDelta = calculateDeltaVector(weightsMatrixSet[l], currentActivationsVector, nextLayerDelta);
-
- // collect delta vectors for this example
- deltaVectors[l - 1] = nextLayerDelta;
+ RealMatrix[] derivatives = new RealMatrix[triangle.length];
+ for (int i = 0; i < triangle.length; i++) {
+ // TODO : introduce regularization diversification on bias term (currently not regularized)
+ derivatives[i] = triangle[i].scalarMultiply(1d / count);
}
+ return derivatives;
}
- private void updateTriangle(RealMatrix[] triangle, RealVector[] activations, RealVector[] deltaVectors) {
- for (int l = triangle.length - 1; l >= 0; l--) {
+ private void updateTriangle(RealMatrix[] triangle, RealVector[] activations, RealVector[] deltaVectors, RealMatrix[] weightsMatrixSet) {
+ for (int l = weightsMatrixSet.length - 1; l >= 0; l--) {
RealMatrix realMatrix = deltaVectors[l].outerProduct(activations[l]);
if (triangle[l] == null) {
triangle[l] = realMatrix;
@@ -126,7 +99,7 @@ class DefaultDerivativeUpdateFunction im
private RealVector calculateDeltaVector(RealMatrix thetaL, RealVector activationsVector, RealVector nextLayerDelta) {
// TODO : remove the bias term from the error calculations
- RealVector identity = new OpenMapRealVector(activationsVector.getDimension(), 1d);
+ ArrayRealVector identity = new ArrayRealVector(activationsVector.getDimension(), 1d);
RealVector gz = activationsVector.ebeMultiply(identity.subtract(activationsVector)); // = a^l .* (1-a^l)
return thetaL.preMultiply(nextLayerDelta).ebeMultiply(gz);
}
@@ -134,12 +107,19 @@ class DefaultDerivativeUpdateFunction im
private RealVector calculateOutputError(TrainingExample<Double, Double> trainingExample, RealVector[] activations) {
RealVector output = activations[activations.length - 1];
+// Double[] sampleOutput = new Double[output.getDimension()];
Double[] actualOutput = trainingExample.getOutput();
- RealVector learnedOutputRealVector = new OpenMapRealVector(actualOutput); // turn example output to a vector
+// int sampleOutputIntValue = actualOutput.intValue();
+// if (sampleOutputIntValue < sampleOutput.length) {
+// sampleOutput[sampleOutputIntValue] = 1d;
+// } else if (sampleOutput.length == 1) {
+// sampleOutput[0] = actualOutput;
+// } else {
+// throw new RuntimeException("problem with multiclass output mapping");
+// }
+ RealVector learnedOutputRealVector = new ArrayRealVector(actualOutput); // turn example output to a vector
- // error calculation -> er_a = out_a * (1 - out_a) * (tgt_a - out_a) (was: output.subtract(learnedOutputRealVector)
- // targetOutputRealVector.subtract(output).map(x -> Math.pow(x, 2)); // squared error
- // return output.subtract(learnedOutputRealVector);
- return output.ebeMultiply(new OpenMapRealVector(output.getDimension(), 1d).subtract(output)).ebeMultiply(output.subtract(learnedOutputRealVector));
+ // TODO : improve error calculation -> this could be er_a = out_a * (1 - out_a) * (tgt_a - out_a)
+ return output.subtract(learnedOutputRealVector);
}
}
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/EncodedTrainingSet.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/EncodedTrainingSet.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/EncodedTrainingSet.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/EncodedTrainingSet.java Thu Dec 17 07:04:12 2015
@@ -18,7 +18,6 @@
*/
package org.apache.yay.core;
-import org.apache.yay.Feature;
import org.apache.yay.TrainingExample;
import org.apache.yay.TrainingSet;
import org.apache.yay.core.utils.ConversionUtils;
@@ -57,12 +56,12 @@ public class EncodedTrainingSet extends
@Override
public TrainingExample<Double, Double> next() {
TrainingExample<Double, Double> sample = EncodedTrainingSet.super.iterator().next();
- Collection<Feature<Double>> features = sample.getFeatures();
+ Collection<Double> features = sample.getFeatures();
int vocabularySize = vocabulary.size();
Double[] outputs = new Double[vocabularySize * (window - 1)];
Double[] inputs = new Double[vocabularySize];
- for (Feature<Double> feature : features) {
- inputs = ConversionUtils.hotEncode(feature.getValue().intValue(), vocabularySize);
+ for (Double feature : features) {
+ inputs = ConversionUtils.hotEncode(feature.intValue(), vocabularySize);
break;
}
int k = 0;
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/FeedForwardStrategy.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/FeedForwardStrategy.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/FeedForwardStrategy.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/FeedForwardStrategy.java Thu Dec 17 07:04:12 2015
@@ -54,20 +54,31 @@ public class FeedForwardStrategy impleme
@Override
public Double[] predictOutput(Collection<Double> input, RealMatrix[] realMatrixSet) {
- RealVector[] activations = applyFF(input, realMatrixSet);
+ RealVector[] activations = debugOutput(input, realMatrixSet);
RealVector x = activations[activations.length - 1];
return ConversionUtils.toDoubleArray(x.toArray());
}
+ @Override
+ public RealVector predictOutput(RealVector inputVector, RealMatrix[] weightsMatrixSet) {
+ RealVector[] activations = debugOutput(inputVector, weightsMatrixSet);
+ return activations[activations.length - 1];
+ }
+
public RealVector[] debugOutput(Collection<Double> input, RealMatrix[] realMatrixSet) {
- return applyFF(input, realMatrixSet);
+ Double[] doubles = input.toArray(new Double[input.size()]);
+ return applyFF(Stream.of(doubles).mapToDouble(Double::doubleValue).toArray(), realMatrixSet);
}
- private RealVector[] applyFF(Collection<Double> input, RealMatrix[] realMatrixSet) {
+ @Override
+ public RealVector[] debugOutput(RealVector inputVector, RealMatrix[] weightsMatrixSet) {
+ return applyFF(inputVector.toArray(), weightsMatrixSet);
+ }
+
+ private RealVector[] applyFF(double[] inputs, RealMatrix[] realMatrixSet) {
RealVector[] debugOutput = new RealVector[realMatrixSet.length];
- Double[] doubles = input.toArray(new Double[input.size()]);
- RealMatrix x = MatrixUtils.createRowRealMatrix(Stream.of(doubles).mapToDouble(Double::doubleValue).toArray());
+ RealMatrix x = MatrixUtils.createRowRealMatrix(inputs);
for (int w = 0; w < realMatrixSet.length; w++) {
// compute matrix multiplication
x = x.multiply(realMatrixSet[w].transpose());
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/LogisticRegressionCostFunction.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/LogisticRegressionCostFunction.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/LogisticRegressionCostFunction.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/LogisticRegressionCostFunction.java Thu Dec 17 07:04:12 2015
@@ -67,13 +67,6 @@ public class LogisticRegressionCostFunct
return res;
}
});
-// for (int i = 0; i < layerMatrix.getColumnDimension(); i++) {
-// double[] column = layerMatrix.getColumn(i);
-// // starting from 1 to avoid including the bias unit in regularization
-// for (int j = 1; j < column.length; j++) {
-// res += Math.pow(column[j], 2d);
-// }
-// }
}
return (lambda / (2d * trainingExamples.size())) * res;
}
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/MaxSelectionFunction.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/MaxSelectionFunction.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/MaxSelectionFunction.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/MaxSelectionFunction.java Thu Dec 17 07:04:12 2015
@@ -18,9 +18,10 @@
*/
package org.apache.yay.core;
+import org.apache.yay.SelectionFunction;
+
import java.util.Collection;
import java.util.Collections;
-import org.apache.yay.SelectionFunction;
/**
* Selects the max value from a {@link Collection} of {@link Comparable} outputs.
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/NeuralNetworkFactory.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/NeuralNetworkFactory.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/NeuralNetworkFactory.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/NeuralNetworkFactory.java Thu Dec 17 07:04:12 2015
@@ -18,7 +18,6 @@
*/
package org.apache.yay.core;
-import java.util.Collection;
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.yay.Input;
import org.apache.yay.LearningException;
@@ -28,7 +27,8 @@ import org.apache.yay.PredictionExceptio
import org.apache.yay.PredictionStrategy;
import org.apache.yay.TrainingSet;
import org.apache.yay.WeightLearningException;
-import org.apache.yay.core.utils.ConversionUtils;
+
+import java.util.Collection;
/**
* Factory class for creating {@link org.apache.yay.NeuralNetwork}s
@@ -49,7 +49,7 @@ public class NeuralNetworkFactory {
return new NeuralNetwork() {
private Double[] getOutputVector(Input<Double> input) {
- Collection<Double> inputVector = ConversionUtils.toValuesCollection(input.getFeatures());
+ Collection<Double> inputVector = input.getFeatures();
return predictionStrategy.predictOutput(inputVector, updatedRealMatrixSet);
}
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ConversionUtils.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ConversionUtils.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ConversionUtils.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ConversionUtils.java Thu Dec 17 07:04:12 2015
@@ -22,16 +22,13 @@ import org.apache.commons.math3.linear.M
import org.apache.commons.math3.linear.OpenMapRealVector;
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.commons.math3.linear.RealVector;
-import org.apache.yay.Feature;
import org.apache.yay.Input;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.WeakHashMap;
-import java.util.stream.Collectors;
/**
* Temporary class for conversion between model objects and commons-math matrices/vectors
@@ -69,8 +66,8 @@ public class ConversionUtils {
private static double[] toDoubleArray(Input<Double> sample) {
double[] ar = new double[sample.getFeatures().size()];
int i = 0;
- for (Feature<Double> f : sample.getFeatures()) {
- ar[i] = f.getValue();
+ for (Double f : sample.getFeatures()) {
+ ar[i] = f;
i++;
}
return ar;
@@ -87,18 +84,6 @@ public class ConversionUtils {
}
/**
- * turns a collection of features of type <code>T</code> into a collection of
- * <code>T</code> objects.
- *
- * @param featureVector the vector of features
- * @param <T> the type of features
- * @return a vector of Doubles
- */
- public static <T> Collection<T> toValuesCollection(Collection<Feature<T>> featureVector) {
- return featureVector.stream().map(Feature::getValue).collect(Collectors.toCollection(ArrayList::new));
- }
-
- /**
* this is just nice! :-) (thanks commons-math)
*
* @param ar a double array
Modified: labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ExamplesFactory.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ExamplesFactory.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ExamplesFactory.java (original)
+++ labs/yay/trunk/core/src/main/java/org/apache/yay/core/utils/ExamplesFactory.java Thu Dec 17 07:04:12 2015
@@ -18,11 +18,11 @@
*/
package org.apache.yay.core.utils;
-import org.apache.yay.Feature;
-import org.apache.yay.Input;
import org.apache.yay.TrainingExample;
-import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
/**
* Factory class for {@link org.apache.yay.Input}s and {@link TrainingExample}s.
@@ -33,8 +33,11 @@ public class ExamplesFactory {
final Double... featuresValues) {
return new TrainingExample<Double, Double>() {
@Override
- public ArrayList<Feature<Double>> getFeatures() {
- return doublesToFeatureVector(featuresValues);
+ public List<Double> getFeatures() {
+ List<Double> doubles = new LinkedList<>();
+ doubles.add(1d);
+ doubles.addAll(Arrays.asList(featuresValues));
+ return doubles;
}
@Override
@@ -45,11 +48,14 @@ public class ExamplesFactory {
}
public static TrainingExample<Double, Double> createDoubleArrayTrainingExample(final Double[] output,
- final Double... featuresValues) {
+ final Double... featuresValues) {
return new TrainingExample<Double, Double>() {
@Override
- public ArrayList<Feature<Double>> getFeatures() {
- return doublesToFeatureVector(featuresValues);
+ public List<Double> getFeatures() {
+ List<Double> doubles = new LinkedList<>();
+ doubles.add(1d);
+ doubles.addAll(Arrays.asList(featuresValues));
+ return doubles;
}
@Override
@@ -59,21 +65,4 @@ public class ExamplesFactory {
};
}
- public static Input<Double> createDoubleInput(final Double... featuresValues) {
- return () -> doublesToFeatureVector(featuresValues);
- }
-
- private static ArrayList<Feature<Double>> doublesToFeatureVector(Double[] featuresValues) {
- ArrayList<Feature<Double>> features = new ArrayList<>();
- Feature<Double> byasFeature = new Feature<>();
- byasFeature.setValue(1d);
- features.add(byasFeature);
- for (Double d : featuresValues) {
- Feature<Double> feature = new Feature<>();
- feature.setValue(d);
- features.add(feature);
- }
- return features;
- }
-
}
Modified: labs/yay/trunk/core/src/test/java/org/apache/yay/core/BasicPerceptronTest.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/java/org/apache/yay/core/BasicPerceptronTest.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/java/org/apache/yay/core/BasicPerceptronTest.java (original)
+++ labs/yay/trunk/core/src/test/java/org/apache/yay/core/BasicPerceptronTest.java Thu Dec 17 07:04:12 2015
@@ -18,16 +18,17 @@
*/
package org.apache.yay.core;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.LinkedList;
-import java.util.Random;
-import org.apache.yay.Feature;
import org.apache.yay.TrainingExample;
import org.apache.yay.TrainingSet;
import org.junit.Before;
import org.junit.Test;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Random;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -143,16 +144,11 @@ public class BasicPerceptronTest {
final Double... params) {
return new TrainingExample<Double, Double>() {
@Override
- public ArrayList<Feature<Double>> getFeatures() {
- ArrayList<Feature<Double>> features = new ArrayList<>();
- Feature<Double> byasFeature = new Feature<>();
- byasFeature.setValue(1d);
+ public List<Double> getFeatures() {
+ List<Double> features = new LinkedList<>();
+ Double byasFeature = 1d;
features.add(byasFeature);
- for (Double d : params) {
- Feature<Double> feature = new Feature<>();
- feature.setValue(d);
- features.add(feature);
- }
+ Collections.addAll(features, params);
return features;
}
Modified: labs/yay/trunk/core/src/test/java/org/apache/yay/core/NeuralNetworkIntegrationTest.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/java/org/apache/yay/core/NeuralNetworkIntegrationTest.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/java/org/apache/yay/core/NeuralNetworkIntegrationTest.java (original)
+++ labs/yay/trunk/core/src/test/java/org/apache/yay/core/NeuralNetworkIntegrationTest.java Thu Dec 17 07:04:12 2015
@@ -22,12 +22,19 @@ import org.apache.commons.math3.linear.A
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.commons.math3.ml.distance.CanberraDistance;
import org.apache.commons.math3.ml.distance.DistanceMeasure;
-import org.apache.yay.*;
+import org.apache.yay.Input;
+import org.apache.yay.LearningStrategy;
+import org.apache.yay.NeuralNetwork;
+import org.apache.yay.TrainingExample;
+import org.apache.yay.TrainingSet;
import org.apache.yay.core.utils.ExamplesFactory;
import org.junit.Test;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Random;
import static org.junit.Assert.assertEquals;
@@ -105,15 +112,10 @@ public class NeuralNetworkIntegrationTes
private Input<Double> createSample(final Double... params) {
return () -> {
- ArrayList<Feature<Double>> features = new ArrayList<Feature<Double>>();
- Feature<Double> byasFeature = new Feature<Double>();
- byasFeature.setValue(1d);
+ List<Double> features = new LinkedList<>();
+ Double byasFeature = 1d;
features.add(byasFeature);
- for (Double d : params) {
- Feature<Double> feature = new Feature<Double>();
- feature.setValue(d);
- features.add(feature);
- }
+ Collections.addAll(features, params);
return features;
};
}
Modified: labs/yay/trunk/core/src/test/java/org/apache/yay/core/WordVectorsTest.java
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/java/org/apache/yay/core/WordVectorsTest.java?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/java/org/apache/yay/core/WordVectorsTest.java (original)
+++ labs/yay/trunk/core/src/test/java/org/apache/yay/core/WordVectorsTest.java Thu Dec 17 07:04:12 2015
@@ -23,9 +23,7 @@ import org.apache.commons.math3.linear.M
import org.apache.commons.math3.linear.RealMatrix;
import org.apache.commons.math3.ml.distance.DistanceMeasure;
import org.apache.commons.math3.ml.distance.EuclideanDistance;
-import org.apache.commons.math3.util.FastMath;
import org.apache.yay.ActivationFunction;
-import org.apache.yay.Feature;
import org.apache.yay.NeuralNetwork;
import org.apache.yay.TrainingExample;
import org.apache.yay.TrainingSet;
@@ -78,7 +76,7 @@ public class WordVectorsTest {
Path path = Paths.get(getClass().getResource("/word2vec/test.txt").getFile());
System.out.println("reading fragments");
- int window = 4;
+ int window = 3;
Queue<List<byte[]>> fragments = getFragments(path, window);
assertFalse(fragments.isEmpty());
System.out.println("generating vocabulary");
@@ -93,7 +91,7 @@ public class WordVectorsTest {
int inputSize = next.getFeatures().size();
int outputSize = next.getOutput().length;
- int hiddenSize = 30;
+ int hiddenSize = 10;
System.out.println("initializing neural network");
RealMatrix[] randomWeights = createRandomWeights(inputSize, hiddenSize, outputSize);
@@ -128,31 +126,31 @@ public class WordVectorsTest {
System.out.println("measuring similarities");
Collection<DistanceMeasure> measures = new LinkedList<>();
measures.add(new EuclideanDistance());
- measures.add(new DistanceMeasure() {
- @Override
- public double compute(double[] a, double[] b) {
- double dp = 0.0;
- double na = 0.0;
- double nb = 0.0;
- for (int i = 0; i < a.length; i++) {
- dp += a[i] * b[i];
- na += Math.pow(a[i], 2);
- nb += Math.pow(b[i], 2);
- }
- double cosineSimilarity = dp / (Math.sqrt(na) * Math.sqrt(nb));
- return 1 / cosineSimilarity;
- }
-
- @Override
- public String toString() {
- return "inverse cosine similarity distance measure";
- }
- });
- measures.add((DistanceMeasure) (a, b) -> {
- double da = FastMath.sqrt(MatrixUtils.createRealVector(a).dotProduct(MatrixUtils.createRealVector(a)));
- double db = FastMath.sqrt(MatrixUtils.createRealVector(b).dotProduct(MatrixUtils.createRealVector(b)));
- return Math.abs(db - da);
- });
+// measures.add(new DistanceMeasure() {
+// @Override
+// public double compute(double[] a, double[] b) {
+// double dp = 0.0;
+// double na = 0.0;
+// double nb = 0.0;
+// for (int i = 0; i < a.length; i++) {
+// dp += a[i] * b[i];
+// na += Math.pow(a[i], 2);
+// nb += Math.pow(b[i], 2);
+// }
+// double cosineSimilarity = dp / (Math.sqrt(na) * Math.sqrt(nb));
+// return 1 / cosineSimilarity;
+// }
+//
+// @Override
+// public String toString() {
+// return "inverse cosine similarity distance measure";
+// }
+// });
+// measures.add((DistanceMeasure) (a, b) -> {
+// double da = FastMath.sqrt(MatrixUtils.createRealVector(a).dotProduct(MatrixUtils.createRealVector(a)));
+// double db = FastMath.sqrt(MatrixUtils.createRealVector(b).dotProduct(MatrixUtils.createRealVector(b)));
+// return Math.abs(db - da);
+// });
for (DistanceMeasure distanceMeasure : measures) {
System.out.println("computing similarity using " + distanceMeasure);
computeSimilarities(vocabulary, wordVectors, distanceMeasure);
@@ -255,10 +253,9 @@ public class WordVectorsTest {
}
@Override
- public ArrayList<Feature<Double>> getFeatures() {
- ArrayList<Feature<Double>> features = new ArrayList<>();
- Feature<Double> e = new Feature<>();
- e.setValue((double) vocabulary.indexOf(new String(finalInputWord)));
+ public List<Double> getFeatures() {
+ List<Double> features = new ArrayList<>();
+ Double e = (double) vocabulary.indexOf(new String(finalInputWord));
features.add(e);
return features;
}
@@ -275,7 +272,6 @@ public class WordVectorsTest {
private List<String> getVocabulary(Path path) throws IOException {
- long start = System.currentTimeMillis();
Set<String> vocabulary = new HashSet<>();
ByteBuffer buf = ByteBuffer.allocate(100);
try (SeekableByteChannel sbc = Files.newByteChannel(path)) {
@@ -307,16 +303,17 @@ public class WordVectorsTest {
} finally {
buf.clear();
}
- long end = System.currentTimeMillis();
List<String> list = Arrays.asList(vocabulary.toArray(new String[vocabulary.size()]));
Collections.sort(list);
- System.out.println("vocabulary read in " + (end - start) / 60000 + " minutes (" + (list.size()) + ")");
+// for (String iw : vocabulary) {
+// System.out.println(iw +"->"+Arrays.toString(ConversionUtils.hotEncode(iw.getBytes(), list)));
+// }
return list;
}
private String cleanString(CharBuffer charBuffer) {
String s = charBuffer.toString();
- return s.toLowerCase().replaceAll("\\.", " ").replaceAll("\\;", " ").replaceAll("\\,", " ").replaceAll("\\:", " ").replaceAll("\\-","").replaceAll("\\\"","");
+ return s.toLowerCase().replaceAll("\\.", " ").replaceAll("\\;", " ").replaceAll("\\,", " ").replaceAll("\\:", " ").replaceAll("\\-\\s", "").replaceAll("\\\"", "");
}
private List<String> getVocabulary(Collection<byte[]> sentences) {
Modified: labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt
URL: http://svn.apache.org/viewvc/labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt?rev=1720488&r1=1720487&r2=1720488&view=diff
==============================================================================
--- labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt (original)
+++ labs/yay/trunk/core/src/test/resources/word2vec/abstracts.txt Thu Dec 17 07:04:12 2015
@@ -1,10 +1,10 @@
-A calculus which combined the flexible geometric structure of vector mod- els with the crisp efficiency of Boolean logic would be extremely beneficial for modelling natural language. With this goal in mind, we present a formulation for logical connectives in vector spaces based on standard linear algebra, giving ex- amples of the use of vector negation to discriminate between different senses of ambiguous words. It turns out that the operators developed in this way are pre- cisely the connectives of quantum logic (Birkhoff and von Neumann, 1936), which to our knowledge have not been exploited before in natural language processing. In quantum logic, arbitrary sets are replaced by linear subspaces of a vector space, and set unions, intersections and complements are replaced by vector sum, inter- section and orthogonal complements of subspaces. We demonstrate that these logi- cal connectives (particularly the orthogonal complement for negation) are powerful tools for exploring and anal
ysing word meanings and show distinct advantages over Boolean operators in document retrieval experiments.
+A calculus which combined the flexible geometric structure of vector models with the crisp efficiency of Boolean logic would be extremely beneficial for modelling natural language. With this goal in mind, we present a formulation for logical connectives in vector spaces based on standard linear algebra, giving ex- amples of the use of vector negation to discriminate between different senses of ambiguous words. It turns out that the operators developed in this way are pre- cisely the connectives of quantum logic (Birkhoff and von Neumann, 1936), which to our knowledge have not been exploited before in natural language processing. In quantum logic, arbitrary sets are replaced by linear subspaces of a vector space, and set unions, intersections and complements are replaced by vector sum, inter- section and orthogonal complements of subspaces. We demonstrate that these logi- cal connectives (particularly the orthogonal complement for negation) are powerful tools for exploring and analys
ing word meanings and show distinct advantages over Boolean operators in document retrieval experiments.
This paper is organised as follows. In Section 1.1 we describe some of the ways vectors have been used to represent the meanings of terms and documents in natural language processing, and describe the way the WORD-SPACE used in our later experiments is built automatically from text corpora. In Section 1.2 we define the logical connectives on vector spaces, focussing particularly on negation and disjunction. This introduces the basic material needed to understand the worked examples given in Section 1.3, and the document retrieval experiments described in Section 1.3.1. Section 1.4 gives a much fuller outline of the theory of quantum logic, the natural setting for the operators of Section 1.2. Finally, in Section 1.5, we examine the similarities between quantum logic and WORD-SPACE, asking whether quantum logic is an appropriate framework for modelling word-meanings or if the
initial successes we have obtained are mainly coincidental.
To some extent, this paper may have been written backwards, in that the im-plementation and examples are at the beginning and most of the theory is at the end. This is for two reasons. Firstly, we hoped to make the paper as accessible as possible and were afraid that beginning with an introduction to the full machinery of quantum logic would defeat this goal before the reader has a chance to realise that the techniques and equations used in this work are really quite elementary. Secondly, the link with âquantum logicâ was itself only brought to our attention after the bulk of the results in this paper had been obtained, and since this research is very much ongoing, we deemed it appropriate to give an honest account of its history and current state.
-We propose two novel model architectures for computing continuous vector repre- sentations of words from very large data sets. The quality of these representations is measured in a word similarity task, and the results are compared to the previ- ously best performing techniques based on different types of neural networks. We observe large improvements in accuracy at much lower computational cost, i.e. it takes less than a day to learn high quality word vectors from a 1.6 billion words data set. Furthermore, we show that these vectors provide state-of-the-art perfor- mance on our test set for measuring syntactic and semantic word similarities.
+We propose two novel model architectures for computing continuous vector representations of words from very large data sets. The quality of these representations is measured in a word similarity task, and the results are compared to the previ- ously best performing techniques based on different types of neural networks. We observe large improvements in accuracy at much lower computational cost, i.e. it takes less than a day to learn high quality word vectors from a 1.6 billion words data set. Furthermore, we show that these vectors provide state-of-the-art perfor- mance on our test set for measuring syntactic and semantic word similarities.
Information Retrieval (IR) models need to deal with two difficult issues, vocabulary mismatch and term dependencies. Vocabulary mismatch corresponds to the difficulty of retrieving relevant documents that do not contain exact query terms but semantically related terms. Term dependencies refers to the need of considering the relationship between the words of the query when estimating the relevance of a document. A multitude of solutions has been proposed to solve each of these two problems, but no principled model solve both. In parallel, in the last few years, language models based on neural networks have been used to cope with complex natural language processing tasks like emotion and paraphrase detection. Although they present good abilities to cope with both term dependencies and vocabulary mismatch problems, thanks to the distributed representation of words they are based upon, such models could not be used readily in IR, where the estimation of one language model per document (
or query) is required. This is both computationally unfeasible and prone to over-fitting. Based on a recent work that proposed to learn a generic language model that can be modified through a set of document-specific parameters, we explore use of new neural network models that are adapted to ad-hoc IR tasks. Within the language model IR framework, we propose and study the use of a generic language model as well as a document-specific language model. Both can be used as a smoothing component, but the latter is more adapted to the document at hand and has the potential of being used as a full document language model. We experiment with such models and analyze their results on TREC-1 to 8 datasets.
-Bidirectional Long Short-Term Mem- ory Recurrent Neural Network (BLSTM- RNN) has been shown to be very effec- tive for modeling and predicting sequen- tial data, e.g. speech utterances or hand- written documents. In this study, we propose to use BLSTM-RNN for a uni- fied tagging solution that can be applied to various tagging tasks including part- of-speech tagging, chunking and named entity recognition. Instead of exploiting specific features carefully optimized for each task, our solution only uses one set of task-independent features and internal representations learnt from unlabeled text for all tasks. Requiring no task specific knowledge or sophisticated feature engi- neering, our approach gets nearly state-of- the-art performance in all these three tag- ging tasks.
+Bidirectional Long Short-Term Memory Recurrent Neural Network (BLSTM-RNN) has been shown to be very effec- tive for modeling and predicting sequen- tial data, e.g. speech utterances or hand- written documents. In this study, we propose to use BLSTM-RNN for a uni- fied tagging solution that can be applied to various tagging tasks including part- of-speech tagging, chunking and named entity recognition. Instead of exploiting specific features carefully optimized for each task, our solution only uses one set of task-independent features and internal representations learnt from unlabeled text for all tasks. Requiring no task specific knowledge or sophisticated feature engi- neering, our approach gets nearly state-of- the-art performance in all these three tag- ging tasks.
The recently introduced continuous Skip-gram model is an efficient method for learning high-quality distributed vector representations that capture a large num- ber of precise syntactic and semantic word relationships. In this paper we present several extensions that improve both the quality of the vectors and the training speed. By subsampling of the frequent words we obtain significant speedup and also learn more regular word representations. We also describe a simple alterna- tive to the hierarchical softmax called negative sampling.
An inherent limitation of word representations is their indifference to word order and their inability to represent idiomatic phrases. For example, the meanings of âCanadaâ and âAirâ cannot be easily combined to obtain âAir Canadaâ. Motivated by this example, we present a simple method for finding phrases in text, and show that learning good vector representations for millions of phrases is possible.
We extend the word2vec framework to capture meaning across languages. The input consists of a source text and a word-aligned parallel text in a second language. The joint word2vec tool then repre- sents words in both languages within a common âsemanticâ vector space. The result can be used to enrich lexicons of under-resourced languages, to identify ambiguities, and to perform clustering and classification. Experiments were conducted on a parallel English-Arabic corpus, as well as on English and Hebrew Biblical texts.
@@ -13,7 +13,7 @@ We report our participation in the conte
We present a comprehensive study of eval- uation methods for unsupervised embed- ding techniques that obtain meaningful representations of words from text. Differ- ent evaluations result in different orderings of embedding methods, calling into ques- tion the common assumption that there is one single optimal vector representation. We present new evaluation techniques that directly compare embeddings with respect to specific queries. These methods re- duce bias, provide greater insight, and allow us to solicit data-driven relevance judgments rapidly and accurately through crowdsourcing.
Continuous word and phrase vectors have proven useful in a number of NLP tasks. Here we describe our experience using them as a source of features for the SemEval-2015 task 3, consisting of two community question an- swering subtasks: Answer Selection for cate- gorizing answers as potential, good, and bad with regards to their corresponding questions; and YES/NO inference for predicting a yes, no, or unsure response to a YES/NO question us- ing all of its good answers. Our system ranked 6th and 1st in the English answer selection and YES/NO inference subtasks respectively, and 2nd in the Arabic answer selection subtask.
The word2vec model and application by Mikolov et al. have attracted a great amount of attention in recent two years. The vector representations of words learned by word2vec models have been proven to be able to carry semantic meanings and are useful in various NLP tasks. As an increasing number of researchers would like to experiment with word2vec, I notice that there lacks a material that comprehensively explains the parameter learning process of word2vec in details, thus preventing many people with less neural network experience from understanding how exactly word2vec works.
-This note provides detailed derivations and explanations of the parameter up- date equations for the word2vec models, including the original continuous bag-of-word (CBOW) and skip-gram models, as well as advanced tricks, hierarchical soft-max and negative sampling. In the appendix a review is given on the basics of neuron network models and backpropagation.
+This note provides detailed derivations and explanations of the parameter update equations for the word2vec models, including the original continuous bag-of-word (CBOW) and skip-gram models, as well as advanced tricks, hierarchical soft-max and negative sampling. In the appendix a review is given on the basics of neuron network models and backpropagation.
Over the past few years, neural networks have re-emerged as powerful machine-learning
models, yielding state-of-the-art results in fields such as image recognition and speech
processing. More recently, neural network models started to be applied also to textual
@@ -22,4 +22,13 @@ network models from the perspective of n
to bring natural-language researchers up to speed with the neural techniques. The tutorial
covers input encoding for natural language tasks, feed-forward networks, convolutional
networks, recurrent networks and recursive networks, as well as the computation graph
-abstraction for automatic gradient computation
\ No newline at end of file
+abstraction for automatic gradient computation
+The development of intelligent machines is one of the biggest unsolved
+challenges in computer science. In this paper, we propose some
+fundamental properties these machines should have, focusing in particular
+on communication and learning. We discuss a simple environment
+that could be used to incrementally teach a machine the basics
+of natural-language-based communication, as a prerequisite to more
+complex interaction with human users. We also present some conjectures
+on the sort of algorithms the machine should support in order
+to profitably learn from the environment.
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@labs.apache.org
For additional commands, e-mail: commits-help@labs.apache.org