You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2010/06/28 11:41:11 UTC
svn commit: r958517 [2/2] - in /mahout/trunk/core/src:
main/java/org/apache/mahout/cf/taste/hadoop/
main/java/org/apache/mahout/cf/taste/hadoop/similarity/
main/java/org/apache/mahout/cf/taste/hadoop/similarity/item/
main/java/org/apache/mahout/math/ha...
Added: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarity.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarity.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarity.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarity.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.hadoop.similarity.Cooccurrence;
+
+/**
+ * distributed implementation of the tanimoto coefficient
+ */
+public class DistributedTanimotoCoefficientVectorSimilarity extends AbstractDistributedVectorSimilarity {
+
+ @Override
+ protected double doComputeResult(int rowA, int rowB, Iterable<Cooccurrence> cooccurrences, double weightOfVectorA,
+ double weightOfVectorB, int numberOfColumns) {
+ double cooccurrenceCount = countElements(cooccurrences);
+ if (cooccurrenceCount == 0) {
+ return Double.NaN;
+ }
+ return cooccurrenceCount / (weightOfVectorA + weightOfVectorB - cooccurrenceCount);
+ }
+
+ @Override
+ public double weight(Vector v) {
+ return (double) countElements(v.iterateNonZero());
+ }
+}
Added: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarity.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarity.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarity.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarity.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,41 @@
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+import org.apache.mahout.math.hadoop.similarity.Cooccurrence;
+
+/**
+ * distributed implementation of cosine similarity that does not center its data
+ */
+public class DistributedUncenteredCosineVectorSimilarity extends AbstractDistributedVectorSimilarity {
+
+ @Override
+ protected double doComputeResult(int rowA, int rowB, Iterable<Cooccurrence> cooccurrences, double weightOfVectorA,
+ double weightOfVectorB, int numberOfColumns) {
+
+ int n = 0;
+ double sumXY = 0.0;
+ double sumX2 = 0.0;
+ double sumY2 = 0.0;
+
+ for (Cooccurrence cooccurrence : cooccurrences) {
+ double x = cooccurrence.getValueA();
+ double y = cooccurrence.getValueB();
+
+ sumXY += x * y;
+ sumX2 += x * x;
+ sumY2 += y * y;
+ n++;
+ }
+
+ if (n == 0) {
+ return Double.NaN;
+ }
+ double denominator = Math.sqrt(sumX2) * Math.sqrt(sumY2);
+ if (denominator == 0.0) {
+ // One or both vectors has -all- the same values;
+ // can't really say much similarity under this measure
+ return Double.NaN;
+ }
+ return sumXY / denominator;
+ }
+
+}
Added: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarity.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarity.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarity.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarity.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+import java.util.Iterator;
+
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.Vector.Element;
+import org.apache.mahout.math.hadoop.similarity.Cooccurrence;
+
+/**
+ * distributed implementation of the cosine similarity of two vectors
+ */
+public class DistributedUncenteredZeroAssumingCosineVectorSimilarity extends AbstractDistributedVectorSimilarity {
+
+ @Override
+ protected double doComputeResult(int rowA, int rowB, Iterable<Cooccurrence> cooccurrences, double weightOfVectorA,
+ double weightOfVectorB, int numberOfColumns) {
+
+ double sumXY = 0.0;
+ for (Cooccurrence cooccurrence : cooccurrences) {
+ sumXY += cooccurrence.getValueA() * cooccurrence.getValueB();
+ }
+
+ if (sumXY == 0.0) {
+ return Double.NaN;
+ }
+ return sumXY / (weightOfVectorA * weightOfVectorB);
+ }
+
+ @Override
+ public double weight(Vector v) {
+ double length = 0.0;
+ Iterator<Element> elemIterator = v.iterateNonZero();
+ while (elemIterator.hasNext()) {
+ double value = elemIterator.next().get();
+ length += value * value;
+ }
+ return Math.sqrt(length);
+ }
+
+}
Added: mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarity.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarity.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarity.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarity.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.hadoop.similarity.Cooccurrence;
+
+/**
+ * a measure for the pairwise similarity of two rows of a matrix that is suitable for computing that similarity
+ * in a distributed way
+ *
+ * works in 2 steps:
+ * - at first weight() is called for each of the row vectors
+ * - later similarity is called with the previously computed weights as parameters
+ *
+ */
+public interface DistributedVectorSimilarity {
+
+ /**
+ * compute the weight (e.g. length) of a vector
+ * @param v
+ * @return
+ */
+ double weight(Vector v);
+
+ /**
+ * compute the similarity of a pair of row vectors
+ *
+ * @param rowA offset of the first row
+ * @param rowB offset of the second row
+ * @param cooccurrences all column entries where both vectors have a nonZero entry
+ * @param weightOfVectorA the result of {@link DistributedVectorSimilarity#weight(Vector)} for the first row vector
+ * @param weightOfVectorB the result of {@link DistributedVectorSimilarity#weight(Vector)} for the first row vector
+ * @param numberOfCols the overall number of columns
+ * @return
+ */
+ double similarity(int rowA, int rowB, Iterable<Cooccurrence> cooccurrences, double weightOfVectorA,
+ double weightOfVectorB, int numberOfColumns);
+}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityTest.java?rev=958517&r1=958516&r2=958517&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/hadoop/similarity/item/ItemSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -18,55 +18,48 @@
package org.apache.mahout.cf.taste.hadoop.similarity.item;
import java.io.BufferedReader;
-import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
-import java.io.FileWriter;
import java.io.FilenameFilter;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.mahout.cf.taste.hadoop.EntityEntityWritable;
-import org.apache.mahout.cf.taste.hadoop.EntityPrefWritable;
-import org.apache.mahout.cf.taste.hadoop.EntityPrefWritableArrayWritable;
-import org.apache.mahout.cf.taste.hadoop.ToUserPrefsMapper;
-import org.apache.mahout.cf.taste.hadoop.similarity.CoRating;
-import org.apache.mahout.cf.taste.hadoop.similarity.DistributedTanimotoCoefficientSimilarity;
-import org.apache.mahout.cf.taste.hadoop.similarity.DistributedUncenteredZeroAssumingCosineSimilarity;
-import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.cf.taste.hadoop.TasteHadoopUtils;
+import org.apache.mahout.cf.taste.impl.TasteTestCase;
+import org.apache.mahout.math.RandomAccessSparseVector;
import org.apache.mahout.math.VarIntWritable;
import org.apache.mahout.math.VarLongWritable;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.VectorWritable;
+import org.apache.mahout.math.hadoop.DistributedRowMatrix;
+import org.apache.mahout.math.hadoop.MathHelper;
+import org.apache.mahout.math.hadoop.DistributedRowMatrix.MatrixEntryWritable;
+import org.apache.mahout.math.hadoop.similarity.vector.DistributedTanimotoCoefficientVectorSimilarity;
+import org.apache.mahout.math.hadoop.similarity.vector.DistributedUncenteredZeroAssumingCosineVectorSimilarity;
+import org.apache.mahout.math.map.OpenIntLongHashMap;
import org.easymock.IArgumentMatcher;
import org.easymock.classextension.EasyMock;
/**
- * Unit tests for the mappers and reducers in org.apache.mahout.cf.taste.hadoop.similarity
- * Integration test with a mini-file at the end
+ * Unit tests for the mappers and reducers in org.apache.mahout.cf.taste.hadoop.similarity.item
+ * some integration tests with tiny data sets at the end
*/
-public final class ItemSimilarityTest extends MahoutTestCase {
-
- public void testUserPrefsPerItemMapper() throws Exception {
- Mapper<LongWritable,Text,VarLongWritable,VarLongWritable>.Context context =
- EasyMock.createMock(Mapper.Context.class);
- context.write(new VarLongWritable(34L), new EntityPrefWritable(12L, 2.3f));
- EasyMock.replay(context);
-
- new ToUserPrefsMapper().map(new LongWritable(), new Text("12,34,2.3"), context);
-
- EasyMock.verify(context);
- }
+public final class ItemSimilarityTest extends TasteTestCase {
+ /**
+ * tests {@link CountUsersMapper}
+ * @throws Exception
+ */
public void testCountUsersMapper() throws Exception {
Mapper<LongWritable,Text,CountUsersKeyWritable,VarLongWritable>.Context context =
EasyMock.createMock(Mapper.Context.class);
@@ -81,6 +74,12 @@ public final class ItemSimilarityTest ex
EasyMock.verify(context);
}
+ /**
+ * applies an {@link IArgumentMatcher} to a {@link CountUsersKeyWritable} checking whether it matches the userID
+ *
+ * @param userID
+ * @return
+ */
static CountUsersKeyWritable keyForUserID(final long userID) {
EasyMock.reportMatcher(new IArgumentMatcher() {
@Override
@@ -99,8 +98,12 @@ public final class ItemSimilarityTest ex
return null;
}
+ /**
+ * tests {@link CountUsersReducer}
+ *
+ * @throws Exception
+ */
public void testCountUsersReducer() throws Exception {
-
Reducer<CountUsersKeyWritable,VarLongWritable,VarIntWritable,NullWritable>.Context context =
EasyMock.createMock(Reducer.Context.class);
context.write(new VarIntWritable(3), NullWritable.get());
@@ -111,237 +114,118 @@ public final class ItemSimilarityTest ex
new VarLongWritable(5L), new VarLongWritable(5L));
new CountUsersReducer().reduce(null, userIDs, context);
-
EasyMock.verify(context);
}
- public void testToItemVectorReducer() throws Exception {
-
- List<EntityPrefWritable> userPrefs = Arrays.asList(
- new EntityPrefWritable(34L, 1.0f), new EntityPrefWritable(56L, 2.0f));
-
- Reducer<VarLongWritable,EntityPrefWritable,VarLongWritable,EntityPrefWritableArrayWritable>.Context context =
- EasyMock.createMock(Reducer.Context.class);
-
- context.write(EasyMock.eq(new VarLongWritable(12L)), equalToUserPrefs(userPrefs));
-
+ /**
+ * tests {@link PrefsToItemUserMatrixMapper}
+ *
+ * @throws Exception
+ */
+ public void testPrefsToItemUserMatrixMapper() throws Exception {
+ Mapper<LongWritable,Text,VarIntWritable,DistributedRowMatrix.MatrixEntryWritable>.Context context =
+ EasyMock.createMock(Mapper.Context.class);
+ context.write(EasyMock.eq(new VarIntWritable(TasteHadoopUtils.idToIndex(100L))),
+ MathHelper.matrixEntryMatches(TasteHadoopUtils.idToIndex(100L),
+ TasteHadoopUtils.idToIndex(12L), 1.3d));
+ context.write(EasyMock.eq(new VarIntWritable(TasteHadoopUtils.idToIndex(20L))),
+ MathHelper.matrixEntryMatches(TasteHadoopUtils.idToIndex(20L), TasteHadoopUtils.idToIndex(35L), 3.0d));
EasyMock.replay(context);
- new ToItemVectorReducer().reduce(new VarLongWritable(12L), userPrefs, context);
+ PrefsToItemUserMatrixMapper mapper = new PrefsToItemUserMatrixMapper();
+ mapper.map(null, new Text("12,100,1.3"), context);
+ mapper.map(null, new Text("35,20,3.0"), context);
EasyMock.verify(context);
}
+ /**
+ * tests {@link PrefsToItemUserMatrixReducer}
+ *
+ * @throws Exception
+ */
+ public void testPrefsToItemUserMatrixReducer() throws Exception {
+ Reducer<VarIntWritable,DistributedRowMatrix.MatrixEntryWritable,IntWritable,VectorWritable>.Context context =
+ EasyMock.createMock(Reducer.Context.class);
- static EntityPrefWritableArrayWritable equalToUserPrefs(
- final Collection<EntityPrefWritable> prefsToCheck) {
- EasyMock.reportMatcher(new IArgumentMatcher() {
- @Override
- public boolean matches(Object argument) {
- if (argument instanceof EntityPrefWritableArrayWritable) {
- EntityPrefWritableArrayWritable userPrefArray =
- (EntityPrefWritableArrayWritable) argument;
- Set<EntityPrefWritable> set = new HashSet<EntityPrefWritable>();
- set.addAll(Arrays.asList(userPrefArray.getPrefs()));
-
- if (set.size() != prefsToCheck.size()) {
- return false;
- }
-
- for (EntityPrefWritable prefToCheck : prefsToCheck) {
- if (!set.contains(prefToCheck)) {
- return false;
- }
- }
- return true;
- }
- return false;
- }
-
- @Override
- public void appendTo(StringBuffer buffer) {}
- });
-
- return null;
- }
-
- public void testPreferredItemsPerUserMapper() throws Exception {
- Mapper<VarLongWritable,EntityPrefWritableArrayWritable,VarLongWritable,ItemPrefWithItemVectorWeightWritable>.Context context =
- EasyMock.createMock(Mapper.Context.class);
- EntityPrefWritableArrayWritable userPrefs = new EntityPrefWritableArrayWritable(
- new EntityPrefWritable[] {
- new EntityPrefWritable(12L, 2.0f),
- new EntityPrefWritable(56L, 3.0f) });
-
- Configuration conf = new Configuration();
- EasyMock.expect(context.getConfiguration()).andStubReturn(conf);
-
- double weight =
- new DistributedUncenteredZeroAssumingCosineSimilarity().weightOfItemVector(Arrays.asList(2.0f, 3.0f));
-
- context.write(new VarLongWritable(12L), new ItemPrefWithItemVectorWeightWritable(34L, weight, 2.0f));
- context.write(new VarLongWritable(56L), new ItemPrefWithItemVectorWeightWritable(34L, weight, 3.0f));
-
- conf.set(ItemSimilarityJob.DISTRIBUTED_SIMILARITY_CLASSNAME,
- "org.apache.mahout.cf.taste.hadoop.similarity.DistributedUncenteredZeroAssumingCosineSimilarity");
-
+ context.write(EasyMock.eq(new IntWritable(123)), MathHelper.vectorMatches(MathHelper.elem(1, 0.5d),
+ MathHelper.elem(7, 2d)));
EasyMock.replay(context);
- PreferredItemsPerUserMapper mapper = new PreferredItemsPerUserMapper();
- mapper.setup(context);
- mapper.map(new VarLongWritable(34L), userPrefs, context);
-
- EasyMock.verify(context);
- }
-
- public void testPreferredItemsPerUserReducer() throws Exception {
-
- List<ItemPrefWithItemVectorWeightWritable> itemPrefs =
- Arrays.asList(new ItemPrefWithItemVectorWeightWritable(34L, 5.0, 1.0f),
- new ItemPrefWithItemVectorWeightWritable(56L, 7.0, 2.0f));
+ List<MatrixEntryWritable> entries = Arrays.asList(MathHelper.matrixEntry(123, 1, 0.5d),
+ MathHelper.matrixEntry(123, 7, 2d));
- Reducer<VarLongWritable,ItemPrefWithItemVectorWeightWritable,VarLongWritable,ItemPrefWithItemVectorWeightArrayWritable>.Context context =
- EasyMock.createMock(Reducer.Context.class);
-
- context.write(EasyMock.eq(new VarLongWritable(12L)), equalToItemPrefs(itemPrefs));
-
- EasyMock.replay(context);
-
- new PreferredItemsPerUserReducer().reduce(new VarLongWritable(12L), itemPrefs, context);
+ new PrefsToItemUserMatrixReducer().reduce(new VarIntWritable(123), entries, context);
EasyMock.verify(context);
}
- static ItemPrefWithItemVectorWeightArrayWritable equalToItemPrefs(
- final Collection<ItemPrefWithItemVectorWeightWritable> prefsToCheck) {
- EasyMock.reportMatcher(new IArgumentMatcher() {
- @Override
- public boolean matches(Object argument) {
- if (argument instanceof ItemPrefWithItemVectorWeightArrayWritable) {
- ItemPrefWithItemVectorWeightArrayWritable itemPrefArray = (ItemPrefWithItemVectorWeightArrayWritable) argument;
- Collection<ItemPrefWithItemVectorWeightWritable> set = new HashSet<ItemPrefWithItemVectorWeightWritable>();
- set.addAll(Arrays.asList(itemPrefArray.getItemPrefs()));
-
- if (set.size() != prefsToCheck.size()) {
- return false;
- }
-
- for (ItemPrefWithItemVectorWeightWritable prefToCheck : prefsToCheck) {
- if (!set.contains(prefToCheck)) {
- return false;
- }
- }
- return true;
- }
- return false;
- }
-
- @Override
- public void appendTo(StringBuffer buffer) {}
- });
-
- return null;
- }
-
- public void testCopreferredItemsMapper() throws Exception {
- Mapper<VarLongWritable,ItemPrefWithItemVectorWeightArrayWritable,ItemPairWritable,CoRating>.Context context =
- EasyMock.createMock(Mapper.Context.class);
- ItemPrefWithItemVectorWeightArrayWritable itemPrefs =
- EasyMock.createMock(ItemPrefWithItemVectorWeightArrayWritable.class);
-
- EasyMock.expect(itemPrefs.getItemPrefs()).andReturn(new ItemPrefWithItemVectorWeightWritable[] {
- new ItemPrefWithItemVectorWeightWritable(34L, 2.0, 1.0f), new ItemPrefWithItemVectorWeightWritable(56L, 3.0, 2.0f),
- new ItemPrefWithItemVectorWeightWritable(78L, 4.0, 3.0f) });
-
- context.write(new ItemPairWritable(34L, 56L, 2.0, 3.0), new CoRating(1.0f, 2.0f));
- context.write(new ItemPairWritable(34L, 78L, 2.0, 4.0), new CoRating(1.0f, 3.0f));
- context.write(new ItemPairWritable(56L, 78L, 3.0, 4.0), new CoRating(2.0f, 3.0f));
-
- EasyMock.replay(context, itemPrefs);
-
- new CopreferredItemsMapper().map(new VarLongWritable(), itemPrefs, context);
-
- EasyMock.verify(context, itemPrefs);
- }
-
- public void testSimilarityReducer() throws Exception {
- Reducer<ItemPairWritable,CoRating,EntityEntityWritable,DoubleWritable>.Context context =
- EasyMock.createMock(Reducer.Context.class);
- Configuration conf = new Configuration();
- EasyMock.expect(context.getConfiguration()).andStubReturn(conf);
-
- conf.set(ItemSimilarityJob.DISTRIBUTED_SIMILARITY_CLASSNAME,
- "org.apache.mahout.cf.taste.hadoop.similarity.DistributedUncenteredZeroAssumingCosineSimilarity");
- conf.setInt(ItemSimilarityJob.NUMBER_OF_USERS, 1);
-
- context.write(new EntityEntityWritable(12L, 34L), new DoubleWritable(0.5));
+ /**
+ * tests {@link MostSimilarItemPairsMapper}
+ *
+ * @throws Exception
+ */
+ public void testMostSimilarItemsPairsMapper() throws Exception {
+
+ OpenIntLongHashMap indexItemIDMap = new OpenIntLongHashMap();
+ indexItemIDMap.put(12, 12L);
+ indexItemIDMap.put(34, 34L);
+ indexItemIDMap.put(56, 56L);
- EasyMock.replay(context);
-
- SimilarityReducer reducer = new SimilarityReducer();
- reducer.setup(context);
- reducer.reduce(new ItemPairWritable(12L, 34L, 2.0, 10.0),
- Arrays.asList(new CoRating(2.5f, 2.0f), new CoRating(2.0f, 2.5f)), context);
-
- EasyMock.verify(context);
- }
-
- public void testCapSimilaritiesPerItemMapper() throws Exception {
- Mapper<EntityEntityWritable,DoubleWritable,CapSimilaritiesPerItemKeyWritable,SimilarItemWritable>.Context context =
+ Mapper<IntWritable,VectorWritable,EntityEntityWritable,DoubleWritable>.Context context =
EasyMock.createMock(Mapper.Context.class);
- context.write(new CapSimilaritiesPerItemKeyWritable(1L, 0.89d), new SimilarItemWritable(5L, 0.89d));
- context.write(new CapSimilaritiesPerItemKeyWritable(5L, 0.89d), new SimilarItemWritable(1L, 0.89d));
-
- EasyMock.replay(context);
-
- CapSimilaritiesPerItemMapper mapper = new CapSimilaritiesPerItemMapper();
- EntityEntityWritable itemPair = new EntityEntityWritable(1L, 5L);
- mapper.map(itemPair, new DoubleWritable(0.89d), context);
-
- EasyMock.verify(context);
- }
-
- public void testCapSimilaritiesPerItemReducer() throws Exception {
- Reducer<CapSimilaritiesPerItemKeyWritable,SimilarItemWritable,EntityEntityWritable,DoubleWritable>.Context context =
- EasyMock.createMock(Reducer.Context.class);
-
- Configuration conf = new Configuration();
- EasyMock.expect(context.getConfiguration()).andStubReturn(conf);
- conf.setInt(ItemSimilarityJob.MAX_SIMILARITIES_PER_ITEM, 2);
-
- context.write(new EntityEntityWritable(1L, 3L), new DoubleWritable(0.9d));
- context.write(new EntityEntityWritable(1L, 6L), new DoubleWritable(0.7d));
+ context.write(new EntityEntityWritable(34L, 56L), new DoubleWritable(0.9d));
EasyMock.replay(context);
- CapSimilaritiesPerItemReducer reducer = new CapSimilaritiesPerItemReducer();
-
- List<SimilarItemWritable> similarItems = Arrays.asList(new SimilarItemWritable(3L, 0.9d),
- new SimilarItemWritable(6L, 0.7d), new SimilarItemWritable(123l, 0.2d));
+ Vector vector = new RandomAccessSparseVector(Integer.MAX_VALUE);
+ vector.set(12, 0.2d);
+ vector.set(34, 1d);
+ vector.set(56, 0.9d);
+
+ MostSimilarItemPairsMapper mapper = new MostSimilarItemPairsMapper();
+ setField(mapper, "indexItemIDMap", indexItemIDMap);
+ setField(mapper, "maxSimilarItemsPerItem", 1);
- reducer.setup(context);
- reducer.reduce(new CapSimilaritiesPerItemKeyWritable(1L, 1d), similarItems, context);
+ mapper.map(new IntWritable(34), new VectorWritable(vector), context);
EasyMock.verify(context);
}
- public void testRemoveDuplicatesReducer() throws Exception {
+ /**
+ * tests {@link MostSimilarItemPairsReducer}
+ *
+ * @throws Exception
+ */
+ public void testMostSimilarItemPairsReducer() throws Exception {
Reducer<EntityEntityWritable,DoubleWritable,EntityEntityWritable,DoubleWritable>.Context context =
EasyMock.createMock(Reducer.Context.class);
- context.write(new EntityEntityWritable(1L, 2L), new DoubleWritable(0.5d));
+ context.write(new EntityEntityWritable(123L, 456L), new DoubleWritable(0.5d));
EasyMock.replay(context);
- List<DoubleWritable> values = Arrays.asList(new DoubleWritable(0.5d), new DoubleWritable(0.5d));
-
- RemoveDuplicatesReducer reducer = new RemoveDuplicatesReducer();
- reducer.reduce(new EntityEntityWritable(1L, 2L), values, context);
+ new MostSimilarItemPairsReducer().reduce(new EntityEntityWritable(123L, 456L),
+ Arrays.asList(new DoubleWritable(0.5d), new DoubleWritable(0.5d)), context);
EasyMock.verify(context);
}
+ /**
+ * integration test with a tiny data set
+ *
+ * <pre>
+ * user-item-matrix
+ *
+ * Game Mouse PC Disk
+ * Jane - 1 2 -
+ * Paul 1 - 1 -
+ * Fred - - - 1
+ * </pre>
+ *
+ * @throws Exception
+ */
public void testCompleteJob() throws Exception {
File inputFile = getTestTempFile("prefs.txt");
@@ -349,24 +233,12 @@ public final class ItemSimilarityTest ex
outputDir.delete();
File tmpDir = getTestTempDir("tmp");
- /* user-item-matrix
-
- Game Mouse PC Disk
- Jane - 1 2 -
- Paul 1 - 1 -
- Fred - - - 1
- */
-
- BufferedWriter writer = new BufferedWriter(new FileWriter(inputFile));
- try {
- writer.write("2,1,1\n" +
- "1,2,1\n" +
- "3,4,1\n" +
- "1,3,2\n" +
- "2,3,1\n");
- } finally {
- writer.close();
- }
+ writeLines(inputFile,
+ "2,1,1",
+ "1,2,1",
+ "3,4,1",
+ "1,3,2",
+ "2,3,1");
ItemSimilarityJob similarityJob = new ItemSimilarityJob();
@@ -378,7 +250,7 @@ public final class ItemSimilarityTest ex
similarityJob.setConf(conf);
similarityJob.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--similarityClassname",
- DistributedUncenteredZeroAssumingCosineSimilarity.class.getName()});
+ DistributedUncenteredZeroAssumingCosineVectorSimilarity.class.getName() });
File countUsersPart = new File(tmpDir, "countUsers");
int numberOfUsers = ItemSimilarityJob.readNumberOfUsers(new Configuration(),
@@ -423,6 +295,33 @@ public final class ItemSimilarityTest ex
assertEquals(2, linesWritten);
}
+ /**
+ * integration test for the limitation of the number of computed similarities
+ *
+ * <pre>
+ * user-item-matrix
+ *
+ * i1 i2 i3
+ * u1 1 0 1
+ * u2 0 1 1
+ * u3 1 1 0
+ * u4 1 1 1
+ * u5 0 1 0
+ * u6 1 1 0
+ *
+ * tanimoto(i1,i2) = 0.5
+ * tanimoto(i2,i3) = 0.333
+ * tanimoto(i3,i1) = 0.4
+ *
+ * When we set maxSimilaritiesPerItem to 1 the following pairs should be found:
+ *
+ * i1 --> i2
+ * i2 --> i1
+ * i3 --> i1
+ * </pre>
+ *
+ * @throws Exception
+ */
public void testMaxSimilaritiesPerItem() throws Exception {
File inputFile = getTestTempFile("prefsForMaxSimilarities.txt");
@@ -430,47 +329,21 @@ public final class ItemSimilarityTest ex
outputDir.delete();
File tmpDir = getTestTempDir("tmp");
- /* user-item-matrix
+ writeLines(inputFile,
+ "1,1,1",
+ "1,3,1",
+ "2,2,1",
+ "2,3,1",
+ "3,1,1",
+ "3,2,1",
+ "4,1,1",
+ "4,2,1",
+ "4,3,1",
+ "5,2,1",
+ "6,1,1",
+ "6,2,1");
- i1 i2 i3
- u1 1 0 1
- u2 0 1 1
- u3 1 1 0
- u4 1 1 1
- u5 0 1 0
- u6 1 1 0
-
- tanimoto(i1,i2) = 0.5
- tanimoto(i2,i3) = 0.333
- tanimoto(i3,i1) = 0.4
-
- When we set maxSimilaritiesPerItem to 1 the following pairs should be found:
-
- i1 --> i2
- i2 --> i1
- i3 --> i1
-
- */
-
- BufferedWriter writer = new BufferedWriter(new FileWriter(inputFile));
- try {
- writer.write("1,1,1\n" +
- "1,3,1\n" +
- "2,2,1\n" +
- "2,3,1\n" +
- "3,1,1\n" +
- "3,2,1\n" +
- "4,1,1\n" +
- "4,2,1\n" +
- "4,3,1\n" +
- "5,2,1\n" +
- "6,1,1\n" +
- "6,2,1\n");
- } finally {
- writer.close();
- }
-
- ItemSimilarityJob similarityJob = new ItemSimilarityJob();
+ ItemSimilarityJob similarityJob = new ItemSimilarityJob();
Configuration conf = new Configuration();
conf.set("mapred.input.dir", inputFile.getAbsolutePath());
@@ -480,7 +353,7 @@ public final class ItemSimilarityTest ex
similarityJob.setConf(conf);
similarityJob.run(new String[] { "--tempDir", tmpDir.getAbsolutePath(), "--similarityClassname",
- DistributedTanimotoCoefficientSimilarity.class.getName(), "--maxSimilaritiesPerItem", "1"});
+ DistributedTanimotoCoefficientVectorSimilarity.class.getName(), "--maxSimilaritiesPerItem", "1" });
File outPart = outputDir.listFiles(new FilenameFilter() {
@Override
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java?rev=958517&r1=958516&r2=958517&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/common/MahoutTestCase.java Mon Jun 28 09:41:09 2010
@@ -18,6 +18,7 @@
package org.apache.mahout.common;
import java.io.IOException;
+import java.lang.reflect.Field;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -80,5 +81,9 @@ public abstract class MahoutTestCase ext
return tempFileOrDir;
}
-
+ protected final void setField(Object target, String fieldname, Object value) throws Exception {
+ Field field = target.getClass().getDeclaredField(fieldname);
+ field.setAccessible(true);
+ field.set(target, value);
+ }
}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/MathHelper.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/MathHelper.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/MathHelper.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/MathHelper.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.mahout.math.DenseMatrix;
+import org.apache.mahout.math.Matrix;
+import org.apache.mahout.math.RandomAccessSparseVector;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.VectorWritable;
+import org.apache.mahout.math.Vector.Element;
+import org.apache.mahout.math.hadoop.DistributedRowMatrix.MatrixEntryWritable;
+import org.easymock.IArgumentMatcher;
+import org.easymock.classextension.EasyMock;
+
+/**
+ * a collection of small helper methods useful for unit-testing mathematical operations
+ */
+public class MathHelper {
+
+ /** the "close enough" value for floating point computations */
+ public static final double EPSILON = 0.00001d;
+
+ /**
+ * applies an {@link IArgumentMatcher} to {@link MatrixEntryWritable}s
+ *
+ * @param row
+ * @param col
+ * @param value
+ * @return
+ */
+ public static MatrixEntryWritable matrixEntryMatches(final int row, final int col, final double value) {
+ EasyMock.reportMatcher(new IArgumentMatcher() {
+ @Override
+ public boolean matches(Object argument) {
+ if (argument instanceof MatrixEntryWritable) {
+ MatrixEntryWritable entry = (MatrixEntryWritable) argument;
+ return (row == entry.getRow() && col == entry.getCol() && Math.abs(value - entry.getVal()) <= EPSILON);
+ }
+ return false;
+ }
+
+ @Override
+ public void appendTo(StringBuffer buffer) {}
+ });
+ return null;
+ }
+
+ /**
+ * convenience method to create a {@link MatrixEntryWritable}
+ *
+ * @param row
+ * @param col
+ * @param value
+ * @return
+ */
+ public static MatrixEntryWritable matrixEntry(int row, int col, double value) {
+ MatrixEntryWritable entry = new MatrixEntryWritable();
+ entry.setRow(row);
+ entry.setCol(col);
+ entry.setVal(value);
+ return entry;
+ }
+
+ /**
+ * convenience method to create a {@link Vector.Element}
+ *
+ * @param index
+ * @param value
+ * @return
+ */
+ public static Vector.Element elem(int index, double value) {
+ return new ElementToCheck(index, value);
+ }
+
+ /**
+ * a simple implementation of {@link Vector.Element}
+ */
+ static class ElementToCheck implements Vector.Element {
+ private int index;
+ private double value;
+
+ public ElementToCheck(int index, double value) {
+ super();
+ this.index = index;
+ this.value = value;
+ }
+ @Override
+ public double get() {
+ return value;
+ }
+ @Override
+ public int index() {
+ return index;
+ }
+ @Override
+ public void set(double value) {
+ this.value = value;
+ }
+ }
+
+ /**
+ * applies an {@link IArgumentMatcher} to a {@link VectorWritable} that checks whether all elements are included
+ *
+ * @param elements
+ * @return
+ */
+ public static VectorWritable vectorMatches(final Vector.Element... elements) {
+ EasyMock.reportMatcher(new IArgumentMatcher() {
+ @Override
+ public boolean matches(Object argument) {
+ if (argument instanceof VectorWritable) {
+ Vector v = ((VectorWritable) argument).get();
+ for (Vector.Element element : elements) {
+ boolean matches = Math.abs(element.get() - v.get(element.index())) <= EPSILON;
+ if (!matches) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void appendTo(StringBuffer buffer) {}
+ });
+ return null;
+ }
+
+ /**
+ * read a {@link Matrix} from a SequenceFile<IntWritable,VectorWritable>
+ * @param fs
+ * @param conf
+ * @param path
+ * @param rows
+ * @param columns
+ * @return
+ * @throws IOException
+ */
+ public static Matrix readEntries(FileSystem fs, Configuration conf, Path path, int rows, int columns)
+ throws IOException {
+
+ Matrix matrix = new DenseMatrix(rows, columns);
+
+ SequenceFile.Reader reader = null;
+ try {
+ reader = new SequenceFile.Reader(fs, path, conf);
+ IntWritable key = new IntWritable();
+ VectorWritable value = new VectorWritable();
+ while (reader.next(key, value)) {
+ int row = key.get();
+ Iterator<Element> elementsIterator = value.get().iterateNonZero();
+ while (elementsIterator.hasNext()) {
+ Element element = elementsIterator.next();
+ matrix.set(row, element.index(), element.get());
+ }
+ }
+ } finally {
+ reader.close();
+ }
+ return matrix;
+ }
+
+ /**
+ * write a two-dimensional double array to an SequenceFile<IntWritable,VectorWritable>
+ *
+ * @param entries
+ * @param fs
+ * @param conf
+ * @param path
+ * @throws IOException
+ */
+ public static void writeEntries(double[][] entries, FileSystem fs, Configuration conf, Path path)
+ throws IOException {
+ SequenceFile.Writer writer = null;
+ try {
+ writer = new SequenceFile.Writer(fs, conf, path, IntWritable.class, VectorWritable.class);
+ for (int n = 0; n < entries.length; n++) {
+ Vector v = new RandomAccessSparseVector(entries[n].length);
+ for (int m = 0; m < entries[n].length; m++) {
+ v.setQuick(m, entries[n][m]);
+ }
+ writer.append(new IntWritable(n), new VectorWritable(v));
+ }
+ } finally {
+ writer.close();
+ }
+ }
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestRowSimilarityJob.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestRowSimilarityJob.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestRowSimilarityJob.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/TestRowSimilarityJob.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,393 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.mahout.common.MahoutTestCase;
+import org.apache.mahout.math.DenseMatrix;
+import org.apache.mahout.math.Matrix;
+import org.apache.mahout.math.RandomAccessSparseVector;
+import org.apache.mahout.math.VarIntWritable;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.VectorWritable;
+import org.apache.mahout.math.Vector.Element;
+import org.apache.mahout.math.hadoop.MathHelper;
+import org.apache.mahout.math.hadoop.DistributedRowMatrix.MatrixEntryWritable;
+import org.apache.mahout.math.hadoop.similarity.RowSimilarityJob.EntriesToVectorsReducer;
+import org.apache.mahout.math.hadoop.similarity.RowSimilarityJob.SimilarityReducer;
+import org.apache.mahout.math.hadoop.similarity.vector.DistributedTanimotoCoefficientVectorSimilarity;
+import org.apache.mahout.math.hadoop.similarity.vector.DistributedVectorSimilarity;
+import org.easymock.IArgumentMatcher;
+import org.easymock.classextension.EasyMock;
+
+/**
+ * tests {@link RowSimilarityJob}
+ */
+public class TestRowSimilarityJob extends MahoutTestCase {
+
+ /**
+ * @tests {@link RowSimilarityJob.RowWeightMapper}
+ *
+ * @throws Exception
+ */
+ public void testRowWeightMapper() throws Exception {
+ Mapper<IntWritable,VectorWritable,VarIntWritable,WeightedOccurrence>.Context context =
+ EasyMock.createMock(Mapper.Context.class);
+
+ context.write(new VarIntWritable(456), new WeightedOccurrence(123, 0.5d, 2d));
+ context.write(new VarIntWritable(789), new WeightedOccurrence(123, 0.1d, 2d));
+
+ EasyMock.replay(context);
+
+ Vector vector = new RandomAccessSparseVector(Integer.MAX_VALUE);
+ vector.set(456, 0.5d);
+ vector.set(789, 0.1d);
+
+ RowSimilarityJob.RowWeightMapper mapper = new RowSimilarityJob.RowWeightMapper();
+ setField(mapper, "similarity", new DistributedTanimotoCoefficientVectorSimilarity());
+
+ mapper.map(new IntWritable(123), new VectorWritable(vector), context);
+
+ EasyMock.verify(context);
+ }
+
+ /**
+ * @tests {@link RowSimilarityJob.WeightedOccurrencesPerColumnReducer}
+ *
+ * @throws Exception
+ */
+ public void testWeightedOccurrencesPerColumnReducer() throws Exception {
+
+ List<WeightedOccurrence> weightedOccurrences = Arrays.asList(new WeightedOccurrence(45, 0.5d, 1d),
+ new WeightedOccurrence(78, 3d, 9d));
+
+ Reducer<VarIntWritable,WeightedOccurrence,VarIntWritable,WeightedOccurrenceArray>.Context context =
+ EasyMock.createMock(Reducer.Context.class);
+
+ context.write(EasyMock.eq(new VarIntWritable(123)), weightedOccurrenceArrayMatches(weightedOccurrences));
+
+ EasyMock.replay(context);
+
+ new RowSimilarityJob.WeightedOccurrencesPerColumnReducer().reduce(new VarIntWritable(123), weightedOccurrences,
+ context);
+
+ EasyMock.verify(context);
+ }
+
+ /**
+ * applies an {@link IArgumentMatcher} to a {@link WeightedOccurrenceArray} that checks whether
+ * it matches all {@link WeightedOccurrence}
+ *
+ * @throws Exception
+ */
+ static WeightedOccurrenceArray weightedOccurrenceArrayMatches(
+ final Collection<WeightedOccurrence> occurrencesToMatch) {
+ EasyMock.reportMatcher(new IArgumentMatcher() {
+ @Override
+ public boolean matches(Object argument) {
+ if (argument instanceof WeightedOccurrenceArray) {
+ WeightedOccurrence[] occurrences = ((WeightedOccurrenceArray) argument).getWeightedOccurrences();
+ if (occurrences.length != occurrencesToMatch.size()) {
+ return false;
+ }
+ for (WeightedOccurrence occurrence : occurrences) {
+ if (!occurrencesToMatch.contains(occurrence)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void appendTo(StringBuffer buffer) {}
+ });
+ return null;
+ }
+
+ /**
+ * @tests {@link RowSimilarityJob.CooccurrencesMapper}
+ *
+ * @throws Exception
+ */
+ public void testCooccurrencesMapper() throws Exception {
+ Mapper<VarIntWritable,WeightedOccurrenceArray,WeightedRowPair,Cooccurrence>.Context context =
+ EasyMock.createMock(Mapper.Context.class);
+
+ context.write(new WeightedRowPair(34, 34, 1d, 1d), new Cooccurrence(12, 0.5d, 0.5d));
+ context.write(new WeightedRowPair(34, 56, 1d, 3d), new Cooccurrence(12, 0.5d, 1d));
+ context.write(new WeightedRowPair(56, 56, 3d, 3d), new Cooccurrence(12, 1d, 1d));
+
+ EasyMock.replay(context);
+
+ WeightedOccurrenceArray weightedOccurrences = new WeightedOccurrenceArray(new WeightedOccurrence[] {
+ new WeightedOccurrence(34, 0.5d, 1d), new WeightedOccurrence(56, 1d, 3d) });
+
+ new RowSimilarityJob.CooccurrencesMapper().map(new VarIntWritable(12), weightedOccurrences, context);
+
+ EasyMock.verify(context);
+ }
+
+ /**
+ * @tests {@link RowSimilarityJob.SimilarityReducer}
+ *
+ * @throws Exception
+ */
+ public void testSimilarityReducer() throws Exception {
+
+ Reducer<WeightedRowPair,Cooccurrence,SimilarityMatrixEntryKey,MatrixEntryWritable>.Context context =
+ EasyMock.createMock(Reducer.Context.class);
+
+ context.write(EasyMock.eq(new SimilarityMatrixEntryKey(12, 0.5d)),
+ MathHelper.matrixEntryMatches(12, 34, 0.5d));
+ context.write(EasyMock.eq(new SimilarityMatrixEntryKey(34, 0.5d)),
+ MathHelper.matrixEntryMatches(34, 12, 0.5d));
+
+ EasyMock.replay(context);
+
+ SimilarityReducer reducer = new RowSimilarityJob.SimilarityReducer();
+ setField(reducer, "similarity", new DistributedTanimotoCoefficientVectorSimilarity());
+
+ reducer.reduce(new WeightedRowPair(12, 34, 3d, 3d), Arrays.asList(new Cooccurrence(56, 1d, 2d),
+ new Cooccurrence(78, 3d, 6d)), context);
+
+ EasyMock.verify(context);
+ }
+
+ /**
+ * @tests {@link RowSimilarityJob.SimilarityReducer} in the special case of computing the similarity of a row to
+ * itself
+ *
+ * @throws Exception
+ */
+ public void testSimilarityReducerSelfSimilarity() throws Exception {
+
+ Reducer<WeightedRowPair,Cooccurrence,SimilarityMatrixEntryKey,MatrixEntryWritable>.Context context =
+ EasyMock.createMock(Reducer.Context.class);
+
+ context.write(EasyMock.eq(new SimilarityMatrixEntryKey(90, 1d)), MathHelper.matrixEntryMatches(90, 90, 1d));
+
+ EasyMock.replay(context);
+
+ SimilarityReducer reducer = new RowSimilarityJob.SimilarityReducer();
+ setField(reducer, "similarity", new DistributedTanimotoCoefficientVectorSimilarity());
+
+ reducer.reduce(new WeightedRowPair(90, 90, 2d, 2d), Arrays.asList(new Cooccurrence(56, 1d, 2d),
+ new Cooccurrence(78, 3d, 6d)), context);
+
+ EasyMock.verify(context);
+ }
+
+ /**
+ * @tests {@link RowSimilarityJob.EntriesToVectorsReducer}
+ *
+ * @throws Exception
+ */
+ public void testEntriesToVectorsReducer() throws Exception {
+ Reducer<SimilarityMatrixEntryKey,MatrixEntryWritable,IntWritable,VectorWritable>.Context context =
+ EasyMock.createMock(Reducer.Context.class);
+
+ context.write(EasyMock.eq(new IntWritable(12)), MathHelper.vectorMatches(MathHelper.elem(34, 0.8d)));
+
+ EasyMock.replay(context);
+
+ EntriesToVectorsReducer reducer = new RowSimilarityJob.EntriesToVectorsReducer();
+ setField(reducer, "maxSimilaritiesPerRow", 1);
+
+ reducer.reduce(new SimilarityMatrixEntryKey(12, 1d), Arrays.asList(
+ MathHelper.matrixEntry(12, 34, 0.8d),
+ MathHelper.matrixEntry(12, 56, 0.7d)), context);
+
+ EasyMock.verify(context);
+
+ }
+
+ /**
+ * integration test with a tiny data set
+ *
+ * <pre>
+ *
+ * input matrix:
+ *
+ * 1, 0, 1, 1, 0
+ * 0, 0, 1, 1, 0
+ * 0, 0, 0, 0, 1
+ *
+ * similarity matrix (via tanimoto):
+ *
+ * 1, 0.666, 0
+ * 0.666, 1, 0
+ * 0, 0, 1
+ * </pre>
+ *
+ * @throws Exception
+ */
+ public void testSmallSampleMatrix() throws Exception {
+
+ File inputFile = getTestTempFile("rows");
+ File outputDir = getTestTempDir("output");
+ outputDir.delete();
+ File tmpDir = getTestTempDir("tmp");
+
+ Configuration conf = new Configuration();
+ Path inputPath = new Path(inputFile.getAbsolutePath());
+ FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
+
+ MathHelper.writeEntries(new double[][] {
+ new double[] { 1, 0, 1, 1, 0 },
+ new double[] { 0, 0, 1, 1, 0 },
+ new double[] { 0, 0, 0, 0, 1 }},
+ fs, conf, inputPath);
+
+ conf.set("mapred.input.dir", inputFile.getAbsolutePath());
+ conf.set("mapred.output.dir", outputDir.getAbsolutePath());
+ conf.setBoolean("mapred.output.compress", false);
+
+ RowSimilarityJob rowSimilarityJob = new RowSimilarityJob();
+ rowSimilarityJob.setConf(conf);
+
+ rowSimilarityJob.run(new String[] { "--numberOfColumns", "3", "--similarityClassname",
+ DistributedTanimotoCoefficientVectorSimilarity.class.getName(), "--tempDir", tmpDir.getAbsolutePath() });
+
+ Matrix similarityMatrix =
+ MathHelper.readEntries(fs, conf, new Path(outputDir.getAbsolutePath(), "part-r-00000"), 3, 3);
+
+ assertNotNull(similarityMatrix);
+ assertEquals(3, similarityMatrix.numCols());
+ assertEquals(3, similarityMatrix.numRows());
+
+ assertEquals(1d, similarityMatrix.get(0, 0));
+ assertEquals(1d, similarityMatrix.get(1, 1));
+ assertEquals(1d, similarityMatrix.get(2, 2));
+
+ assertEquals(0d, similarityMatrix.get(2, 0));
+ assertEquals(0d, similarityMatrix.get(2, 1));
+ assertEquals(0d, similarityMatrix.get(0, 2));
+ assertEquals(0d, similarityMatrix.get(1, 2));
+
+ assertEquals(0.6666d, similarityMatrix.get(0, 1), 0.0001);
+ assertEquals(0.6666d, similarityMatrix.get(1, 0), 0.0001);
+ }
+
+ /**
+ * a tanimoto-coefficient like {@link DistributedVectorSimilarity} that returns NaN for identical rows
+ */
+ static class DistributedTanimotoCoefficientExcludeIdentityVectorSimilarity implements DistributedVectorSimilarity {
+
+ private static final DistributedVectorSimilarity tanimoto = new DistributedTanimotoCoefficientVectorSimilarity();
+
+ @Override
+ public double similarity(int rowA, int rowB, Iterable<Cooccurrence> cooccurrences, double weightOfVectorA,
+ double weightOfVectorB, int numberOfRows) {
+ if (rowA == rowB) {
+ return Double.NaN;
+ }
+ return tanimoto.similarity(rowA, rowB, cooccurrences, weightOfVectorA, weightOfVectorB, numberOfRows);
+ }
+
+ @Override
+ public double weight(Vector v) {
+ return tanimoto.weight(v);
+ }
+ }
+
+ /**
+ * integration test for the limitation of the entries of the similarity matrix
+ *
+ * <pre>
+ * c1 c2 c3 c4 c5 c6
+ * r1 1 0 1 1 0 1
+ * r2 0 1 1 1 1 1
+ * r3 1 1 0 1 0 0
+ *
+ * tanimoto(r1,r2) = 0.5
+ * tanimoto(r2,r3) = 0.333
+ * tanimoto(r3,r1) = 0.4
+ *
+ * When we set maxSimilaritiesPerRow to 1 the following pairs should be found:
+ *
+ * r1 --> r2
+ * r2 --> r1
+ * r3 --> r1
+ * </pre>
+ *
+ * @throws Exception
+ */
+ public void testLimitEntriesInSimilarityMatrix() throws Exception {
+
+ File inputFile = getTestTempFile("rows");
+ File outputDir = getTestTempDir("output");
+ outputDir.delete();
+ File tmpDir = getTestTempDir("tmp");
+
+ Configuration conf = new Configuration();
+ Path inputPath = new Path(inputFile.getAbsolutePath());
+ FileSystem fs = FileSystem.get(inputPath.toUri(), conf);
+
+ MathHelper.writeEntries(new double[][] {
+ new double[] { 1, 0, 1, 1, 0, 1 },
+ new double[] { 0, 1, 1, 1, 1, 1 },
+ new double[] { 1, 1, 0, 1, 0, 0 }},
+ fs, conf, inputPath);
+
+ conf.set("mapred.input.dir", inputFile.getAbsolutePath());
+ conf.set("mapred.output.dir", outputDir.getAbsolutePath());
+ conf.setBoolean("mapred.output.compress", false);
+
+ RowSimilarityJob rowSimilarityJob = new RowSimilarityJob();
+ rowSimilarityJob.setConf(conf);
+
+ rowSimilarityJob.run(new String[] { "--numberOfColumns", "3", "--maxSimilaritiesPerRow", "1",
+ "--similarityClassname", DistributedTanimotoCoefficientExcludeIdentityVectorSimilarity.class.getName(),
+ "--tempDir", tmpDir.getAbsolutePath() });
+
+ Matrix similarityMatrix =
+ MathHelper.readEntries(fs, conf, new Path(outputDir.getAbsolutePath(), "part-r-00000"), 3, 3);
+
+ assertNotNull(similarityMatrix);
+ assertEquals(3, similarityMatrix.numCols());
+ assertEquals(3, similarityMatrix.numRows());
+
+ assertEquals(0d, similarityMatrix.get(0, 0));
+ assertEquals(0.5d, similarityMatrix.get(0, 1));
+ assertEquals(0d, similarityMatrix.get(0, 2));
+
+ assertEquals(0.5d, similarityMatrix.get(1, 0));
+ assertEquals(0d, similarityMatrix.get(1, 1));
+ assertEquals(0d, similarityMatrix.get(1, 2));
+
+ assertEquals(0.4d, similarityMatrix.get(2, 0));
+ assertEquals(0d, similarityMatrix.get(2, 1));
+ assertEquals(0d, similarityMatrix.get(2, 2));
+ }
+
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedEuclideanDistanceVectorSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedEuclideanDistanceVectorSimilarityTest.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedEuclideanDistanceVectorSimilarityTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedEuclideanDistanceVectorSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+/**
+ * tests {@link DistributedEuclideanDistanceVectorSimilarity}
+ */
+public class DistributedEuclideanDistanceVectorSimilarityTest extends
+ DistributedVectorSimilarityTestCase {
+
+ public void testEuclideanDistance() throws Exception {
+
+ assertSimilar(new DistributedEuclideanDistanceVectorSimilarity(),
+ asVector(3, -2),
+ asVector(3, -2), 2, 1d);
+
+ assertSimilar(new DistributedEuclideanDistanceVectorSimilarity(),
+ asVector(3, 3),
+ asVector(3, 3), 2, 1d);
+
+ assertSimilar(new DistributedEuclideanDistanceVectorSimilarity(),
+ asVector(1, 2, 3),
+ asVector(2, 5, 6), 3, 0.5598164905901122);
+
+ assertSimilar(new DistributedEuclideanDistanceVectorSimilarity(),
+ asVector(1, 0),
+ asVector(0, 1), 2, 0d);
+ }
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedLoglikelihoodVectorSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedLoglikelihoodVectorSimilarityTest.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedLoglikelihoodVectorSimilarityTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedLoglikelihoodVectorSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+/**
+ * tests {@link DistributedLoglikelihoodVectorSimilarity}
+ */
+public class DistributedLoglikelihoodVectorSimilarityTest
+ extends DistributedVectorSimilarityTestCase {
+
+ public void testLoglikelihood() throws Exception {
+ assertSimilar(new DistributedLoglikelihoodVectorSimilarity(),
+ asVector(1, 1, 0, 1, 0),
+ asVector(1, 0, 0, 1, 1), 5, 0.12160727029227925);
+
+ assertSimilar(new DistributedLoglikelihoodVectorSimilarity(),
+ asVector(1, 0, 0, 1, 1),
+ asVector(1, 1, 0, 1, 0), 5, 0.12160727029227925);
+
+ assertSimilar(new DistributedLoglikelihoodVectorSimilarity(),
+ asVector(1, 0, 0, 1, 1),
+ asVector(0, 1, 1, 1, 1), 5, 0.5423213660693733);
+ }
+
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedPearsonCorrelationVectorSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedPearsonCorrelationVectorSimilarityTest.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedPearsonCorrelationVectorSimilarityTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedPearsonCorrelationVectorSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+/**
+ * tests {@link DistributedPearsonCorrelationVectorSimilarity}
+ */
+public class DistributedPearsonCorrelationVectorSimilarityTest
+ extends DistributedVectorSimilarityTestCase {
+
+ public void testPearsonCorrelation() throws Exception {
+ assertSimilar(new DistributedPearsonCorrelationVectorSimilarity(),
+ asVector(3, -2),
+ asVector(3, -2), 2, 1d);
+
+ assertSimilar(new DistributedPearsonCorrelationVectorSimilarity(),
+ asVector(3, 3),
+ asVector(3, 3), 2, Double.NaN);
+
+ assertSimilar(new DistributedPearsonCorrelationVectorSimilarity(),
+ asVector(0, 3),
+ asVector(3, 0), 2, Double.NaN);
+ }
+
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarityTest.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarityTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedTanimotoCoefficientVectorSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+/**
+ * tests {@link DistributedTanimotoCoefficientVectorSimilarity}
+ */
+public class DistributedTanimotoCoefficientVectorSimilarityTest extends
+ DistributedVectorSimilarityTestCase {
+
+ public void testTanimoto() throws Exception {
+
+ assertSimilar(new DistributedTanimotoCoefficientVectorSimilarity(),
+ asVector(0, 0, 0, 0, 1),
+ asVector(0, 1, 1, 1, 1), 5, 0.25);
+
+ assertSimilar(new DistributedTanimotoCoefficientVectorSimilarity(),
+ asVector(0, 1),
+ asVector(1, 0), 2, Double.NaN);
+
+ assertSimilar(new DistributedTanimotoCoefficientVectorSimilarity(),
+ asVector(0, 1),
+ asVector(0, 1), 2, 1d);
+ }
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarityTest.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarityTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredCosineVectorSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+/**
+ * tests {@link DistributedUncenteredCosineVectorSimilarity}
+ */
+public class DistributedUncenteredCosineVectorSimilarityTest
+ extends DistributedVectorSimilarityTestCase {
+
+ public void testUncenteredCosineSimilarity() throws Exception {
+
+ assertSimilar(new DistributedUncenteredCosineVectorSimilarity(),
+ asVector(0, 0, 0, 0, 1),
+ asVector(0, 1, 1, 1, 1), 5, 1d);
+
+ assertSimilar(new DistributedUncenteredCosineVectorSimilarity(),
+ asVector(0, 1),
+ asVector(1, 0), 2, Double.NaN);
+
+ assertSimilar(new DistributedUncenteredCosineVectorSimilarity(),
+ asVector(1, 0),
+ asVector(1, 0), 2, 1d);
+
+ assertSimilar(new DistributedUncenteredCosineVectorSimilarity(),
+ asVector(1, 1, 2),
+ asVector(3, 5, 0), 3, 0.970142);
+ }
+
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarityTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarityTest.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarityTest.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedUncenteredZeroAssumingCosineVectorSimilarityTest.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+/**
+ * tests {@link DistributedUncenteredZeroAssumingCosineVectorSimilarity}
+ */
+public class DistributedUncenteredZeroAssumingCosineVectorSimilarityTest
+ extends DistributedVectorSimilarityTestCase {
+
+ public void testUncenteredZeroAssumingCosine() throws Exception {
+ assertSimilar(new DistributedUncenteredZeroAssumingCosineVectorSimilarity(),
+ asVector(0, 0, 0, 0, 1),
+ asVector(0, 1, 1, 1, 1), 5, 0.5d);
+
+ assertSimilar(new DistributedUncenteredZeroAssumingCosineVectorSimilarity(),
+ asVector(0, 1),
+ asVector(1, 0), 2, Double.NaN);
+
+ assertSimilar(new DistributedUncenteredZeroAssumingCosineVectorSimilarity(),
+ asVector(1, 0),
+ asVector(1, 0), 2, 1d);
+ }
+}
Added: mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarityTestCase.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarityTestCase.java?rev=958517&view=auto
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarityTestCase.java (added)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/math/hadoop/similarity/vector/DistributedVectorSimilarityTestCase.java Mon Jun 28 09:41:09 2010
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.math.hadoop.similarity.vector;
+
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.mahout.math.DenseVector;
+import org.apache.mahout.math.MahoutTestCase;
+import org.apache.mahout.math.Vector;
+import org.apache.mahout.math.hadoop.MathHelper;
+import org.apache.mahout.math.hadoop.similarity.Cooccurrence;
+import org.apache.mahout.math.hadoop.similarity.RowSimilarityJob;
+
+/**
+ * base testcase for all tests of classes implementing {@link DistributedVectorSimilarity}
+ */
+public abstract class DistributedVectorSimilarityTestCase extends MahoutTestCase {
+
+ /**
+ * convenience method to create a {@link Vector}
+ * @param values
+ * @return
+ */
+ static Vector asVector(double... values) {
+ return new DenseVector(values);
+ }
+
+ /**
+ * @see DistributedVectorSimilarityTestCase#assertSimilar(DistributedVectorSimilarity, int, int, Vector, Vector,
+ * int, double)
+ *
+ * @param similarity
+ * @param v1
+ * @param v2
+ * @param numberOfColumns
+ * @param expectedSimilarity
+ */
+ static void assertSimilar(DistributedVectorSimilarity similarity, Vector v1, Vector v2, int numberOfColumns,
+ double expectedSimilarity) {
+ assertSimilar(similarity, 1, 2, v1, v2, numberOfColumns, expectedSimilarity);
+ }
+
+ /**
+ * emulates the way similarities are computed by {@link RowSimilarityJob}
+ *
+ * @param similarity
+ * @param rowA
+ * @param rowB
+ * @param v1
+ * @param v2
+ * @param numberOfColumns
+ * @param expectedSimilarity
+ */
+ static void assertSimilar(DistributedVectorSimilarity similarity, int rowA, int rowB, Vector v1, Vector v2,
+ int numberOfColumns, double expectedSimilarity) {
+
+ double weightA = similarity.weight(v1);
+ double weightB = similarity.weight(v2);
+
+ List<Cooccurrence> cooccurrences = new LinkedList<Cooccurrence>();
+ for (int n = 0; n < numberOfColumns; n++) {
+ double valueA = v1.get(n);
+ double valueB = v2.get(n);
+ if (valueA != 0d && valueB != 0d) {
+ cooccurrences.add(new Cooccurrence(n, valueA, valueB));
+ }
+ }
+
+ double result = similarity.similarity(rowA, rowB, cooccurrences, weightA, weightB, numberOfColumns);
+ assertEquals(expectedSimilarity, result, MathHelper.EPSILON);
+ }
+}