You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/08/11 14:04:38 UTC
svn commit: r803081 [2/7] - in /lucene/mahout/trunk:
core/src/main/java/org/apache/mahout/cf/taste/eval/
core/src/main/java/org/apache/mahout/cf/taste/hadoop/
core/src/main/java/org/apache/mahout/cf/taste/impl/common/
core/src/main/java/org/apache/maho...
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java (from r802574, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingIterator.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingIterator.java&r1=802574&r2=803081&rev=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingIterator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/SamplingLongPrimitiveIterator.java Tue Aug 11 12:04:35 2009
@@ -25,16 +25,16 @@
* Wraps an {@link Iterator} and returns only some subset of the elements that it would, as determined by a sampling
* rate parameter.
*/
-public final class SamplingIterator<T> implements Iterator<T> {
+public final class SamplingLongPrimitiveIterator implements LongPrimitiveIterator {
private static final Random r = RandomUtils.getRandom();
- private final Iterator<? extends T> delegate;
+ private final LongPrimitiveIterator delegate;
private final double samplingRate;
- private T next;
+ private long next;
private boolean hasNext;
- public SamplingIterator(Iterator<? extends T> delegate, double samplingRate) {
+ public SamplingLongPrimitiveIterator(LongPrimitiveIterator delegate, double samplingRate) {
this.delegate = delegate;
this.samplingRate = samplingRate;
this.hasNext = true;
@@ -47,19 +47,32 @@
}
@Override
- public T next() {
+ public Long next() {
+ return nextLong();
+ }
+
+ @Override
+ public long nextLong() {
if (hasNext) {
- T result = next;
+ long result = next;
doNext();
return result;
}
throw new NoSuchElementException();
}
+ @Override
+ public long peek() {
+ if (hasNext) {
+ return next;
+ }
+ throw new NoSuchElementException();
+ }
+
private void doNext() {
boolean found = false;
if (delegate instanceof SkippingIterator) {
- SkippingIterator<? extends T> skippingDelegate = (SkippingIterator<? extends T>) delegate;
+ SkippingIterator<?> skippingDelegate = (SkippingIterator<?>) delegate;
int toSkip = 0;
while (r.nextDouble() >= samplingRate) {
toSkip++;
@@ -69,12 +82,12 @@
skippingDelegate.skip(toSkip);
}
if (skippingDelegate.hasNext()) {
- next = skippingDelegate.next();
+ next = delegate.next();
found = true;
}
} else {
while (delegate.hasNext()) {
- T delegateNext = delegate.next();
+ long delegateNext = delegate.next();
if (r.nextDouble() < samplingRate) {
next = delegateNext;
found = true;
@@ -84,7 +97,6 @@
}
if (!found) {
hasNext = false;
- next = null;
}
}
@@ -96,4 +108,8 @@
throw new UnsupportedOperationException();
}
-}
+ public static LongPrimitiveIterator maybeWrapIterator(LongPrimitiveIterator delegate, double samplingRate) {
+ return samplingRate >= 1.0 ? delegate : new SamplingLongPrimitiveIterator(delegate, samplingRate);
+ }
+
+}
\ No newline at end of file
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AbstractDifferenceRecommenderEvaluator.java Tue Aug 11 12:04:35 2009
@@ -18,12 +18,15 @@
package org.apache.mahout.cf.taste.impl.eval;
import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.eval.DataModelBuilder;
import org.apache.mahout.cf.taste.eval.RecommenderBuilder;
import org.apache.mahout.cf.taste.eval.RecommenderEvaluator;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.common.RandomUtils;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericPreference;
+import org.apache.mahout.cf.taste.impl.model.GenericUserPreferenceArray;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.PreferenceArray;
@@ -32,9 +35,7 @@
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.List;
-import java.util.Map;
import java.util.Random;
/** <p>Abstract superclass of a couple implementations, providing shared functionality.</p> */
@@ -50,6 +51,7 @@
@Override
public double evaluate(RecommenderBuilder recommenderBuilder,
+ DataModelBuilder dataModelBuilder,
DataModel dataModel,
double trainingPercentage,
double evaluationPercentage) throws TasteException {
@@ -70,18 +72,21 @@
log.info("Beginning evaluation using " + trainingPercentage + " of " + dataModel);
int numUsers = dataModel.getNumUsers();
- Map<Comparable<?>, Collection<Preference>> trainingUsers =
- new FastMap<Comparable<?>, Collection<Preference>>(1 + (int) (trainingPercentage * (double) numUsers));
- Map<Comparable<?>, Collection<Preference>> testUserPrefs =
- new FastMap<Comparable<?>, Collection<Preference>>(1 + (int) ((1.0 - trainingPercentage) * (double) numUsers));
+ FastByIDMap<PreferenceArray> trainingUsers =
+ new FastByIDMap<PreferenceArray>(1 + (int) (trainingPercentage * (double) numUsers));
+ FastByIDMap<PreferenceArray> testUserPrefs =
+ new FastByIDMap<PreferenceArray>(1 + (int) ((1.0 - trainingPercentage) * (double) numUsers));
- for (Comparable<?> userID : dataModel.getUserIDs()) {
+ LongPrimitiveIterator it = dataModel.getUserIDs();
+ while (it.hasNext()) {
if (random.nextDouble() < evaluationPercentage) {
- processOneUser(trainingPercentage, trainingUsers, testUserPrefs, userID, dataModel);
+ processOneUser(trainingPercentage, trainingUsers, testUserPrefs, it.nextLong(), dataModel);
}
}
- DataModel trainingModel = new GenericDataModel(GenericDataModel.toPrefArrayValues(trainingUsers, true));
+ DataModel trainingModel = dataModelBuilder == null ?
+ new GenericDataModel(trainingUsers) :
+ dataModelBuilder.buildDataModel(trainingUsers);
Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
@@ -91,33 +96,37 @@
}
private void processOneUser(double trainingPercentage,
- Map<Comparable<?>, Collection<Preference>> trainingUsers,
- Map<Comparable<?>, Collection<Preference>> testUserPrefs,
- Comparable<?> userID,
+ FastByIDMap<PreferenceArray> trainingUsers,
+ FastByIDMap<PreferenceArray> testUserPrefs,
+ long userID,
DataModel dataModel) throws TasteException {
- List<Preference> trainingPrefs = new ArrayList<Preference>();
- List<Preference> testPrefs = new ArrayList<Preference>();
+ List<Preference> trainingPrefs = null;
+ List<Preference> testPrefs = null;
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
for (int i = 0; i < size; i++) {
Preference newPref = new GenericPreference(userID, prefs.getItemID(i), prefs.getValue(i));
if (random.nextDouble() < trainingPercentage) {
+ if (trainingPrefs == null) {
+ trainingPrefs = new ArrayList<Preference>(3);
+ }
trainingPrefs.add(newPref);
} else {
+ if (testPrefs == null) {
+ testPrefs = new ArrayList<Preference>(3);
+ }
testPrefs.add(newPref);
}
}
- log.debug("Training against {} preferences", trainingPrefs.size());
- log.debug("Evaluating accuracy of {} preferences", testPrefs.size());
- if (!trainingPrefs.isEmpty()) {
- trainingUsers.put(userID, trainingPrefs);
- if (!testPrefs.isEmpty()) {
- testUserPrefs.put(userID, testPrefs);
+ if (trainingPrefs != null) {
+ trainingUsers.put(userID, new GenericUserPreferenceArray(trainingPrefs));
+ if (testPrefs != null) {
+ testUserPrefs.put(userID, new GenericUserPreferenceArray(testPrefs));
}
}
}
- abstract double getEvaluation(Map<Comparable<?>, Collection<Preference>> testUserPrefs, Recommender recommender)
+ abstract double getEvaluation(FastByIDMap<PreferenceArray> testUserPrefs, Recommender recommender)
throws TasteException;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/AverageAbsoluteDifferenceRecommenderEvaluator.java Tue Aug 11 12:04:35 2009
@@ -20,14 +20,15 @@
import org.apache.mahout.cf.taste.common.NoSuchItemException;
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
import org.apache.mahout.cf.taste.impl.common.RunningAverage;
import org.apache.mahout.cf.taste.model.Preference;
+import org.apache.mahout.cf.taste.model.PreferenceArray;
import org.apache.mahout.cf.taste.recommender.Recommender;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
import java.util.Map;
/**
@@ -41,13 +42,11 @@
private static final Logger log = LoggerFactory.getLogger(AverageAbsoluteDifferenceRecommenderEvaluator.class);
@Override
- double getEvaluation(Map<Comparable<?>, Collection<Preference>> testUserPrefs,
- Recommender recommender)
- throws TasteException {
+ double getEvaluation(FastByIDMap<PreferenceArray> testUserPrefs, Recommender recommender) throws TasteException {
RunningAverage average = new FullRunningAverage();
- for (Map.Entry<Comparable<?>, Collection<Preference>> entry : testUserPrefs.entrySet()) {
+ for (Map.Entry<Long, PreferenceArray> entry : testUserPrefs.entrySet()) {
for (Preference realPref : entry.getValue()) {
- Comparable<?> testUserID = entry.getKey();
+ long testUserID = entry.getKey();
try {
float estimatedPreference =
recommender.estimatePreference(testUserID, realPref.getItemID());
@@ -57,9 +56,9 @@
} catch (NoSuchUserException nsue) {
// It's possible that an item exists in the test data but not training data in which case
// NSEE will be thrown. Just ignore it and move on.
- log.debug("User exists in test data but not training data: {}", testUserID, nsue);
+ log.info("User exists in test data but not training data: {}", testUserID);
} catch (NoSuchItemException nsie) {
- log.debug("Item exists in test data but not training data: {}", realPref.getItemID(), nsie);
+ log.info("Item exists in test data but not training data: {}", realPref.getItemID());
}
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java Tue Aug 11 12:04:35 2009
@@ -19,17 +19,20 @@
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.eval.DataModelBuilder;
import org.apache.mahout.cf.taste.eval.IRStatistics;
import org.apache.mahout.cf.taste.eval.RecommenderBuilder;
import org.apache.mahout.cf.taste.eval.RecommenderIRStatsEvaluator;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
import org.apache.mahout.cf.taste.impl.common.FullRunningAverageAndStdDev;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.common.RandomUtils;
import org.apache.mahout.cf.taste.impl.common.RunningAverage;
import org.apache.mahout.cf.taste.impl.common.RunningAverageAndStdDev;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
+import org.apache.mahout.cf.taste.impl.model.GenericUserPreferenceArray;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.PreferenceArray;
@@ -40,10 +43,8 @@
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Iterator;
import java.util.List;
-import java.util.Map;
import java.util.Random;
/**
@@ -58,8 +59,9 @@
private static final Logger log = LoggerFactory.getLogger(GenericRecommenderIRStatsEvaluator.class);
/**
- * Pass as "relevanceThreshold" argument to {@link #evaluate(RecommenderBuilder, DataModel, Rescorer, int, double,
- * double)} to have it attempt to compute a reasonable threshold. Note that this will impact performance.
+ * Pass as "relevanceThreshold" argument to
+ * {@link #evaluate(RecommenderBuilder, DataModelBuilder, DataModel, Rescorer, int, double, double)}
+ * to have it attempt to compute a reasonable threshold. Note that this will impact performance.
*/
public static final double CHOOSE_THRESHOLD = Double.NaN;
@@ -71,8 +73,9 @@
@Override
public IRStatistics evaluate(RecommenderBuilder recommenderBuilder,
+ DataModelBuilder dataModelBuilder,
DataModel dataModel,
- Rescorer<Comparable<?>> rescorer,
+ Rescorer<Long> rescorer,
int at,
double relevanceThreshold,
double evaluationPercentage) throws TasteException {
@@ -97,10 +100,12 @@
RunningAverage precision = new FullRunningAverage();
RunningAverage recall = new FullRunningAverage();
RunningAverage fallOut = new FullRunningAverage();
- for (Comparable<?> userID : dataModel.getUserIDs()) {
+ LongPrimitiveIterator it = dataModel.getUserIDs();
+ while (it.hasNext()) {
+ long userID = it.nextLong();
if (random.nextDouble() < evaluationPercentage) {
long start = System.currentTimeMillis();
- Collection<Comparable<?>> relevantItemIDs = new FastSet<Comparable<?>>(at);
+ FastIDSet relevantItemIDs = new FastIDSet(at);
PreferenceArray prefs = dataModel.getPreferencesFromUser(userID);
int size = prefs.length();
if (size < 2 * at) {
@@ -118,13 +123,16 @@
}
int numRelevantItems = relevantItemIDs.size();
if (numRelevantItems > 0) {
- Map<Comparable<?>, Collection<Preference>> trainingUsers =
- new FastMap<Comparable<?>, Collection<Preference>>(dataModel.getNumUsers());
- for (Comparable userID2 : dataModel.getUserIDs()) {
- processOtherUser(userID, relevantItemIDs, trainingUsers, userID2, dataModel);
+ FastByIDMap<PreferenceArray> trainingUsers =
+ new FastByIDMap<PreferenceArray>(dataModel.getNumUsers());
+ LongPrimitiveIterator it2 = dataModel.getUserIDs();
+ while (it2.hasNext()) {
+ processOtherUser(userID, relevantItemIDs, trainingUsers, it2.nextLong(), dataModel);
}
- DataModel trainingModel = new GenericDataModel(GenericDataModel.toPrefArrayValues(trainingUsers, true));
+ DataModel trainingModel = dataModelBuilder == null ?
+ new GenericDataModel(trainingUsers) :
+ dataModelBuilder.buildDataModel(trainingUsers);
Recommender recommender = recommenderBuilder.buildRecommender(trainingModel);
try {
@@ -162,25 +170,26 @@
return new IRStatisticsImpl(precision.getAverage(), recall.getAverage(), fallOut.getAverage());
}
- private static void processOtherUser(Comparable<?> id,
- Collection<Comparable<?>> relevantItemIDs,
- Map<Comparable<?>, Collection<Preference>> trainingUsers,
- Comparable<?> userID2,
+ private static void processOtherUser(long id,
+ FastIDSet relevantItemIDs,
+ FastByIDMap<PreferenceArray> trainingUsers,
+ long userID2,
DataModel dataModel) throws TasteException {
PreferenceArray prefs2Array = dataModel.getPreferencesFromUser(userID2);
- List<Preference> prefs2 = new ArrayList<Preference>(prefs2Array.length());
- for (Preference pref : prefs2Array) {
- prefs2.add(pref);
- }
- if (id.equals(userID2)) {
+ if (id == userID2) {
+ List<Preference> prefs2 = new ArrayList<Preference>(prefs2Array.length());
+ for (Preference pref : prefs2Array) {
+ prefs2.add(pref);
+ }
for (Iterator<Preference> iterator = prefs2.iterator(); iterator.hasNext();) {
Preference pref = iterator.next();
if (relevantItemIDs.contains(pref.getItemID())) {
iterator.remove();
}
}
+ prefs2Array = new GenericUserPreferenceArray(prefs2);
}
- trainingUsers.put(userID2, prefs2);
+ trainingUsers.put(userID2, prefs2Array);
}
private static double computeThreshold(PreferenceArray prefs) {
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/LoadEvaluator.java Tue Aug 11 12:04:35 2009
@@ -35,11 +35,10 @@
DataModel dataModel = recommender.getDataModel();
int numUsers = dataModel.getNumUsers();
double sampleRate = 1000.0 / numUsers;
- SamplingIterable<Comparable<?>> userSampler =
- new SamplingIterable<Comparable<?>>(dataModel.getUserIDs(), sampleRate);
+ SamplingIterable<Long> userSampler = new SamplingIterable<Long>(dataModel.getUserIDs(), sampleRate);
RunningAverageAndStdDev recommendationTime = new FullRunningAverageAndStdDev();
int count = 0;
- for (Comparable<?> userID : userSampler) {
+ for (long userID : userSampler) {
long start = System.currentTimeMillis();
recommender.recommend(userID, 10);
long end = System.currentTimeMillis();
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/RMSRecommenderEvaluator.java Tue Aug 11 12:04:35 2009
@@ -19,14 +19,15 @@
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.TasteException;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
import org.apache.mahout.cf.taste.impl.common.FullRunningAverage;
import org.apache.mahout.cf.taste.impl.common.RunningAverage;
import org.apache.mahout.cf.taste.model.Preference;
+import org.apache.mahout.cf.taste.model.PreferenceArray;
import org.apache.mahout.cf.taste.recommender.Recommender;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import java.util.Collection;
import java.util.Map;
/**
@@ -39,13 +40,11 @@
private static final Logger log = LoggerFactory.getLogger(RMSRecommenderEvaluator.class);
@Override
- double getEvaluation(Map<Comparable<?>, Collection<Preference>> testUserPrefs,
- Recommender recommender)
- throws TasteException {
+ double getEvaluation(FastByIDMap<PreferenceArray> testUserPrefs, Recommender recommender) throws TasteException {
RunningAverage average = new FullRunningAverage();
- for (Map.Entry<Comparable<?>, Collection<Preference>> entry : testUserPrefs.entrySet()) {
+ for (Map.Entry<Long, PreferenceArray> entry : testUserPrefs.entrySet()) {
for (Preference realPref : entry.getValue()) {
- Comparable<?> testUserID = entry.getKey();
+ long testUserID = entry.getKey();
try {
float estimatedPreference =
recommender.estimatePreference(testUserID, realPref.getItemID());
@@ -56,7 +55,7 @@
} catch (NoSuchUserException nsee) {
// It's possible that an item exists in the test data but not training data in which case
// NSEE will be thrown. Just ignore it and move on.
- log.info("Element exists in test data but not training data: {}", testUserID, nsee);
+ log.info("User exists in test data but not training data: {}", testUserID);
}
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanItemPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -21,6 +21,7 @@
import org.apache.mahout.cf.taste.model.PreferenceArray;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@@ -34,14 +35,14 @@
*/
public final class BooleanItemPreferenceArray implements PreferenceArray, Serializable {
- private final Comparable<?>[] IDs;
- private Comparable<?> id;
+ private final long[] IDs;
+ private long id;
public BooleanItemPreferenceArray(int size) {
if (size < 1) {
throw new IllegalArgumentException("size is less than 1");
}
- this.IDs = new Comparable<?>[size];
+ this.IDs = new long[size];
}
public int length() {
@@ -69,22 +70,22 @@
}
@Override
- public Comparable<?> getUserID(int i) {
+ public long getUserID(int i) {
return IDs[i];
}
@Override
- public void setUserID(int i, Comparable<?> userID) {
+ public void setUserID(int i, long userID) {
IDs[i] = userID;
}
@Override
- public Comparable<?> getItemID(int i) {
+ public long getItemID(int i) {
return id;
}
@Override
- public void setItemID(int i, Comparable<?> itemID) {
+ public void setItemID(int i, long itemID) {
id = itemID;
}
@@ -100,7 +101,7 @@
@Override
public void sortByUser() {
- selectionSort();
+ Arrays.sort(IDs);
}
@Override
@@ -115,24 +116,6 @@
public void sortByValueReversed() {
}
- private void selectionSort() {
- // I think this sort will prove to be too dumb, but, it's in place and OK for tiny, mostly sorted data
- int max = length();
- for (int i = 0; i < max; i++) {
- int min = i;
- for (int j = i + 1; j < max; j++) {
- if (((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0) {
- min = j;
- }
- }
- if (i != min) {
- Comparable<?> temp1 = IDs[i];
- IDs[i] = IDs[min];
- IDs[min] = temp1;
- }
- }
- }
-
public GenericItemPreferenceArray clone() {
try {
return (GenericItemPreferenceArray) super.clone();
@@ -171,12 +154,12 @@
}
@Override
- public Comparable<?> getUserID() {
+ public long getUserID() {
return BooleanItemPreferenceArray.this.getUserID(i);
}
@Override
- public Comparable<?> getItemID() {
+ public long getItemID() {
return BooleanItemPreferenceArray.this.getItemID(i);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanPreference.java Tue Aug 11 12:04:35 2009
@@ -28,24 +28,21 @@
*/
public final class BooleanPreference implements Preference, Serializable {
- private final Comparable<?> userID;
- private final Comparable<?> itemID;
+ private final long userID;
+ private final long itemID;
- public BooleanPreference(Comparable<?> userID, Comparable<?> itemID) {
- if (userID == null || itemID == null) {
- throw new IllegalArgumentException("itemID is null");
- }
+ public BooleanPreference(long userID, long itemID) {
this.userID = userID;
this.itemID = itemID;
}
@Override
- public Comparable<?> getUserID() {
+ public long getUserID() {
return userID;
}
@Override
- public Comparable<?> getItemID() {
+ public long getItemID() {
return itemID;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/BooleanUserPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -21,6 +21,7 @@
import org.apache.mahout.cf.taste.model.PreferenceArray;
import java.io.Serializable;
+import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@@ -34,14 +35,14 @@
*/
public final class BooleanUserPreferenceArray implements PreferenceArray, Serializable {
- private final Comparable<?>[] IDs;
- private Comparable<?> id;
+ private final long[] IDs;
+ private long id;
public BooleanUserPreferenceArray(int size) {
if (size < 1) {
throw new IllegalArgumentException("size is less than 1");
}
- this.IDs = new Comparable<?>[size];
+ this.IDs = new long[size];
}
public int length() {
@@ -69,22 +70,22 @@
}
@Override
- public Comparable<?> getUserID(int i) {
+ public long getUserID(int i) {
return id;
}
@Override
- public void setUserID(int i, Comparable<?> userID) {
+ public void setUserID(int i, long userID) {
id = userID;
}
@Override
- public Comparable<?> getItemID(int i) {
+ public long getItemID(int i) {
return IDs[i];
}
@Override
- public void setItemID(int i, Comparable<?> itemID) {
+ public void setItemID(int i, long itemID) {
IDs[i] = itemID;
}
@@ -104,7 +105,7 @@
@Override
public void sortByItem() {
- selectionSort();
+ Arrays.sort(IDs);
}
@Override
@@ -115,24 +116,6 @@
public void sortByValueReversed() {
}
- private void selectionSort() {
- // I think this sort will prove to be too dumb, but, it's in place and OK for tiny, mostly sorted data
- int max = length();
- for (int i = 0; i < max; i++) {
- int min = i;
- for (int j = i + 1; j < max; j++) {
- if (((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0) {
- min = j;
- }
- }
- if (i != min) {
- Comparable<?> temp1 = IDs[i];
- IDs[i] = IDs[min];
- IDs[min] = temp1;
- }
- }
- }
-
public GenericItemPreferenceArray clone() {
try {
return (GenericItemPreferenceArray) super.clone();
@@ -171,12 +154,12 @@
}
@Override
- public Comparable<?> getUserID() {
+ public long getUserID() {
return BooleanUserPreferenceArray.this.getUserID(i);
}
@Override
- public Comparable<?> getItemID() {
+ public long getItemID() {
return BooleanUserPreferenceArray.this.getItemID(i);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericBooleanPrefDataModel.java Tue Aug 11 12:04:35 2009
@@ -21,15 +21,16 @@
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveArrayIterator;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.PreferenceArray;
import java.io.Serializable;
-import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -40,10 +41,10 @@
*/
public final class GenericBooleanPrefDataModel implements DataModel, Serializable {
- private final List<Comparable<?>> userIDs;
- private final Map<Comparable<?>, FastSet<Comparable<?>>> preferenceFromUsers;
- private final List<Comparable<?>> itemIDs;
- private final Map<Comparable<?>, FastSet<Comparable<?>>> preferenceForItems;
+ private final long[] userIDs;
+ private final FastByIDMap<FastIDSet> preferenceFromUsers;
+ private final long[] itemIDs;
+ private final FastByIDMap<FastIDSet> preferenceForItems;
/**
* <p>Creates a new {@link GenericDataModel} from the given users (and their preferences). This {@link
@@ -52,34 +53,41 @@
* @param userData users to include
*/
@SuppressWarnings("unchecked")
- public GenericBooleanPrefDataModel(Map<Comparable<?>, FastSet<Comparable<?>>> userData) {
+ public GenericBooleanPrefDataModel(FastByIDMap<FastIDSet> userData) {
if (userData == null) {
throw new IllegalArgumentException("userData is null");
}
this.preferenceFromUsers = userData;
- this.preferenceForItems = new FastMap<Comparable<?>, FastSet<Comparable<?>>>();
- FastSet<Comparable<?>> itemIDSet = new FastSet<Comparable<?>>();
- for (Map.Entry<Comparable<?>, FastSet<Comparable<?>>> entry : preferenceFromUsers.entrySet()) {
- Comparable<?> userID = entry.getKey();
- FastSet<Comparable<?>> itemIDs = entry.getValue();
+ this.preferenceForItems = new FastByIDMap<FastIDSet>();
+ FastIDSet itemIDSet = new FastIDSet();
+ for (Map.Entry<Long, FastIDSet> entry : preferenceFromUsers.entrySet()) {
+ long userID = entry.getKey();
+ FastIDSet itemIDs = entry.getValue();
itemIDSet.addAll(itemIDs);
- for (Comparable<?> itemID : itemIDs) {
- FastSet<Comparable<?>> userIDs = preferenceForItems.get(itemID);
+ LongPrimitiveIterator it = itemIDs.iterator();
+ while (it.hasNext()) {
+ long itemID = it.next();
+ FastIDSet userIDs = preferenceForItems.get(itemID);
if (userIDs == null) {
- userIDs = new FastSet<Comparable<?>>(2);
+ userIDs = new FastIDSet(2);
preferenceForItems.put(itemID, userIDs);
}
userIDs.add(userID);
}
}
- this.itemIDs = new ArrayList<Comparable<?>>(itemIDSet);
- itemIDSet = null;
- Collections.sort((List<? extends Comparable>) this.itemIDs);
+ this.itemIDs = itemIDSet.toArray();
+ itemIDSet = null; // Might help GC -- this is big
+ Arrays.sort(itemIDs);
- this.userIDs = new ArrayList(userData.keySet());
- Collections.sort((List<? extends Comparable>) userIDs);
+ this.userIDs = new long[userData.size()];
+ int i = 0;
+ LongPrimitiveIterator it = userData.keySetIterator();
+ while (it.hasNext()) {
+ userIDs[i++] = it.next();
+ }
+ Arrays.sort(userIDs);
}
@@ -94,40 +102,55 @@
this(toDataMap(dataModel));
}
- private static Map<Comparable<?>, FastSet<Comparable<?>>> toDataMap(DataModel dataModel) throws TasteException {
- Map<Comparable<?>, FastSet<Comparable<?>>> data =
- new FastMap<Comparable<?>, FastSet<Comparable<?>>>(dataModel.getNumUsers());
- for (Comparable<?> userID : dataModel.getUserIDs()) {
+ private static FastByIDMap<FastIDSet> toDataMap(DataModel dataModel) throws TasteException {
+ FastByIDMap<FastIDSet> data = new FastByIDMap<FastIDSet>(dataModel.getNumUsers());
+ LongPrimitiveIterator it = dataModel.getUserIDs();
+ while (it.hasNext()) {
+ long userID = it.nextLong();
data.put(userID, dataModel.getItemIDsFromUser(userID));
}
return data;
}
+ public static FastByIDMap<FastIDSet> toDataMap(FastByIDMap<PreferenceArray> data) throws TasteException {
+ for (Map.Entry<Long, Object> entry : ((FastByIDMap<Object>) (FastByIDMap<?>) data).entrySet()) {
+ PreferenceArray prefArray = (PreferenceArray) entry.getValue();
+ int size = prefArray.length();
+ FastIDSet itemIDs = new FastIDSet(size);
+ for (int i = 0; i < size; i++) {
+ itemIDs.add(prefArray.getItemID(i));
+ }
+ entry.setValue(itemIDs);
+ }
+ return (FastByIDMap<FastIDSet>) (FastByIDMap<?>) data;
+ }
+
@Override
- public Iterable<Comparable<?>> getUserIDs() {
- return userIDs;
+ public LongPrimitiveArrayIterator getUserIDs() {
+ return new LongPrimitiveArrayIterator(userIDs);
}
/** @throws NoSuchUserException if there is no such user */
@Override
- public PreferenceArray getPreferencesFromUser(Comparable<?> userID) throws NoSuchUserException {
- FastSet<Comparable<?>> itemIDs = preferenceFromUsers.get(userID);
+ public PreferenceArray getPreferencesFromUser(long userID) throws NoSuchUserException {
+ FastIDSet itemIDs = preferenceFromUsers.get(userID);
if (itemIDs == null) {
throw new NoSuchUserException();
}
PreferenceArray prefArray = new BooleanUserPreferenceArray(itemIDs.size());
int i = 0;
- for (Comparable<?> itemID : itemIDs) {
+ LongPrimitiveIterator it = itemIDs.iterator();
+ while (it.hasNext()) {
prefArray.setUserID(i, userID);
- prefArray.setItemID(i, itemID);
+ prefArray.setItemID(i, it.next());
i++;
}
return prefArray;
}
@Override
- public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> userID) throws TasteException {
- FastSet<Comparable<?>> itemIDs = preferenceFromUsers.get(userID);
+ public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
+ FastIDSet itemIDs = preferenceFromUsers.get(userID);
if (itemIDs == null) {
throw new NoSuchUserException();
}
@@ -135,20 +158,21 @@
}
@Override
- public Iterable<Comparable<?>> getItemIDs() {
- return itemIDs;
+ public LongPrimitiveArrayIterator getItemIDs() {
+ return new LongPrimitiveArrayIterator(itemIDs);
}
@Override
- public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws NoSuchItemException {
- FastSet<Comparable<?>> userIDs = preferenceForItems.get(itemID);
+ public PreferenceArray getPreferencesForItem(long itemID) throws NoSuchItemException {
+ FastIDSet userIDs = preferenceForItems.get(itemID);
if (userIDs == null) {
throw new NoSuchItemException();
}
PreferenceArray prefArray = new BooleanItemPreferenceArray(userIDs.size());
int i = 0;
- for (Comparable<?> userID : userIDs) {
- prefArray.setUserID(i, userID);
+ LongPrimitiveIterator it = userIDs.iterator();
+ while (it.hasNext()) {
+ prefArray.setUserID(i, it.next());
prefArray.setItemID(i, itemID);
i++;
}
@@ -156,8 +180,8 @@
}
@Override
- public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws NoSuchUserException, NoSuchItemException {
- FastSet<Comparable<?>> itemIDs = preferenceFromUsers.get(userID);
+ public Float getPreferenceValue(long userID, long itemID) throws NoSuchUserException, NoSuchItemException {
+ FastIDSet itemIDs = preferenceFromUsers.get(userID);
if (itemIDs == null) {
throw new NoSuchUserException();
}
@@ -169,24 +193,24 @@
@Override
public int getNumItems() {
- return itemIDs.size();
+ return itemIDs.length;
}
@Override
public int getNumUsers() {
- return userIDs.size();
+ return userIDs.length;
}
@Override
- public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) throws NoSuchItemException {
+ public int getNumUsersWithPreferenceFor(long... itemIDs) throws NoSuchItemException {
if (itemIDs.length == 0) {
return 0;
}
- FastSet<Comparable<?>> intersection = new FastSet<Comparable<?>>();
- FastSet<Comparable<?>> userIDs = preferenceForItems.get(itemIDs[0]);
+ FastIDSet userIDs = preferenceForItems.get(itemIDs[0]);
if (userIDs == null) {
throw new NoSuchItemException();
}
+ FastIDSet intersection = new FastIDSet(userIDs.size());
intersection.addAll(userIDs);
int i = 1;
while (!intersection.isEmpty() && i < itemIDs.length) {
@@ -200,12 +224,12 @@
}
@Override
- public void removePreference(Comparable<?> userID, Comparable<?> itemID) throws NoSuchUserException {
+ public void removePreference(long userID, long itemID) throws NoSuchUserException {
throw new UnsupportedOperationException();
}
@Override
- public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value) throws NoSuchUserException {
+ public void setPreference(long userID, long itemID, float value) throws NoSuchUserException {
throw new UnsupportedOperationException();
}
@@ -216,7 +240,19 @@
@Override
public String toString() {
- return "GenericBooleanPrefDataModel[users:" + (userIDs.size() > 3 ? userIDs.subList(0, 3) + "..." : userIDs) + ']';
+ StringBuilder result = new StringBuilder(200);
+ result.append("GenericDataModel[users:");
+ for (int i = 0; i < Math.min(3, userIDs.length); i++) {
+ if (i > 0) {
+ result.append(',');
+ }
+ result.append(userIDs[i]);
+ }
+ if (result.length() > 3) {
+ result.append("...");
+ }
+ result.append(']');
+ return result.toString();
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericDataModel.java Tue Aug 11 12:04:35 2009
@@ -21,8 +21,10 @@
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveArrayIterator;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Preference;
import org.apache.mahout.cf.taste.model.PreferenceArray;
@@ -31,11 +33,10 @@
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
-import java.util.Set;
/**
* <p>A simple {@link DataModel} which uses a given {@link List} of users as its data source. This
@@ -46,33 +47,33 @@
private static final Logger log = LoggerFactory.getLogger(GenericDataModel.class);
- private final List<Comparable<?>> userIDs;
- private final Map<Comparable<?>, PreferenceArray> preferenceFromUsers;
- private final List<Comparable<?>> itemIDs;
- private final Map<Comparable<?>, PreferenceArray> preferenceForItems;
+ private final long[] userIDs;
+ private final FastByIDMap<PreferenceArray> preferenceFromUsers;
+ private final long[] itemIDs;
+ private final FastByIDMap<PreferenceArray> preferenceForItems;
/**
* <p>Creates a new {@link GenericDataModel} from the given users (and their preferences). This {@link
* DataModel} retains all this information in memory and is effectively immutable.</p>
*
* @param userData users to include in this {@link GenericDataModel}
- * (see also {@link #toPrefArrayValues(Map, boolean)})
+ * (see also {@link #toDataMap(FastByIDMap, boolean)})
*/
@SuppressWarnings("unchecked")
- public GenericDataModel(Map<Comparable<?>, PreferenceArray> userData) {
+ public GenericDataModel(FastByIDMap<PreferenceArray> userData) {
if (userData == null) {
throw new IllegalArgumentException("userData is null");
}
this.preferenceFromUsers = userData;
- FastMap<Comparable<?>, Collection<Preference>> prefsForItems = new FastMap<Comparable<?>, Collection<Preference>>();
- FastSet<Comparable<?>> itemIDSet = new FastSet<Comparable<?>>();
+ FastByIDMap<Collection<Preference>> prefsForItems = new FastByIDMap<Collection<Preference>>();
+ FastIDSet itemIDSet = new FastIDSet();
int currentCount = 0;
- for (PreferenceArray prefs : preferenceFromUsers.values()) {
+ for (Map.Entry<Long, PreferenceArray> entry : preferenceFromUsers.entrySet()) {
+ PreferenceArray prefs = entry.getValue();
prefs.sortByItem();
- int size = prefs.length();
for (Preference preference : prefs) {
- Comparable<?> itemID = preference.getItemID();
+ long itemID = preference.getItemID();
itemIDSet.add(itemID);
List<Preference> prefsForItem = (List<Preference>) prefsForItems.get(itemID);
if (prefsForItem == null) {
@@ -81,24 +82,28 @@
}
prefsForItem.add(preference);
}
- currentCount++;
- if (currentCount % 10000 == 0) {
+ if (++currentCount % 10000 == 0) {
log.info("Processed {} users", currentCount);
}
}
- this.itemIDs = new ArrayList<Comparable<?>>(itemIDSet);
- itemIDSet = null;
- Collections.sort((List<? extends Comparable>) this.itemIDs);
-
- this.preferenceForItems = toPrefArrayValues(prefsForItems, false);
-
- for (PreferenceArray prefs : preferenceForItems.values()) {
- prefs.sortByUser();
- }
-
- this.userIDs = new ArrayList(userData.keySet());
- Collections.sort((List<? extends Comparable>) userIDs);
+ this.itemIDs = itemIDSet.toArray();
+ itemIDSet = null; // Might help GC -- this is big
+ Arrays.sort(itemIDs);
+
+ this.preferenceForItems = toDataMap(prefsForItems, false);
+
+ for (Map.Entry<Long, PreferenceArray> entry : preferenceForItems.entrySet()) {
+ entry.getValue().sortByUser();
+ }
+
+ this.userIDs = new long[userData.size()];
+ int i = 0;
+ LongPrimitiveIterator it = userData.keySetIterator();
+ while (it.hasNext()) {
+ userIDs[i++] = it.next();
+ }
+ Arrays.sort(userIDs);
}
/**
@@ -117,32 +122,32 @@
* .
* @return input value
*/
- public static Map<Comparable<?>, PreferenceArray> toPrefArrayValues(Map<Comparable<?>, Collection<Preference>> data,
- boolean byUser) {
- for (Map.Entry<Comparable<?>, Object> entry :
- ((Map<Comparable<?>, Object>) (Map<Comparable<?>, ?>) data).entrySet()) {
+ public static FastByIDMap<PreferenceArray> toDataMap(FastByIDMap<Collection<Preference>> data, boolean byUser) {
+ for (Map.Entry<Long, Object> entry : ((FastByIDMap<Object>) (FastByIDMap<?>) data).entrySet()) {
List<Preference> prefList = (List<Preference>) entry.getValue();
entry.setValue(byUser ? new GenericUserPreferenceArray(prefList) : new GenericItemPreferenceArray(prefList));
}
- return (Map<Comparable<?>, PreferenceArray>) (Map<Comparable<?>, ?>) data;
+ return (FastByIDMap<PreferenceArray>) (FastByIDMap<?>) data;
}
- private static Map<Comparable<?>, PreferenceArray> toDataMap(DataModel dataModel) throws TasteException {
- Map<Comparable<?>, PreferenceArray> data = new FastMap<Comparable<?>, PreferenceArray>(dataModel.getNumUsers());
- for (Comparable<?> userID : dataModel.getUserIDs()) {
+ private static FastByIDMap<PreferenceArray> toDataMap(DataModel dataModel) throws TasteException {
+ FastByIDMap<PreferenceArray> data = new FastByIDMap<PreferenceArray>(dataModel.getNumUsers());
+ LongPrimitiveIterator it = dataModel.getUserIDs();
+ while (it.hasNext()) {
+ long userID = it.nextLong();
data.put(userID, dataModel.getPreferencesFromUser(userID));
}
return data;
}
@Override
- public Iterable<Comparable<?>> getUserIDs() {
- return userIDs;
+ public LongPrimitiveArrayIterator getUserIDs() {
+ return new LongPrimitiveArrayIterator(userIDs);
}
/** @throws NoSuchUserException if there is no such user */
@Override
- public PreferenceArray getPreferencesFromUser(Comparable<?> userID) throws NoSuchUserException {
+ public PreferenceArray getPreferencesFromUser(long userID) throws NoSuchUserException {
PreferenceArray prefs = preferenceFromUsers.get(userID);
if (prefs == null) {
throw new NoSuchUserException();
@@ -151,10 +156,10 @@
}
@Override
- public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> userID) throws TasteException {
+ public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
PreferenceArray prefs = getPreferencesFromUser(userID);
int size = prefs.length();
- FastSet<Comparable<?>> result = new FastSet<Comparable<?>>(size);
+ FastIDSet result = new FastIDSet(size);
for (int i = 0; i < size; i++) {
result.add(prefs.getItemID(i));
}
@@ -162,12 +167,12 @@
}
@Override
- public Iterable<Comparable<?>> getItemIDs() {
- return itemIDs;
+ public LongPrimitiveArrayIterator getItemIDs() {
+ return new LongPrimitiveArrayIterator(itemIDs);
}
@Override
- public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws NoSuchItemException {
+ public PreferenceArray getPreferencesForItem(long itemID) throws NoSuchItemException {
PreferenceArray prefs = preferenceForItems.get(itemID);
if (prefs == null) {
throw new NoSuchItemException();
@@ -176,11 +181,11 @@
}
@Override
- public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+ public Float getPreferenceValue(long userID, long itemID) throws TasteException {
PreferenceArray prefs = getPreferencesFromUser(userID);
int size = prefs.length();
for (int i = 0; i < size; i++) {
- if (prefs.getItemID(i).equals(itemID)) {
+ if (prefs.getItemID(i) == itemID) {
return prefs.getValue(i);
}
}
@@ -189,16 +194,16 @@
@Override
public int getNumItems() {
- return itemIDs.size();
+ return itemIDs.length;
}
@Override
public int getNumUsers() {
- return userIDs.size();
+ return userIDs.length;
}
@Override
- public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) {
+ public int getNumUsersWithPreferenceFor(long... itemIDs) {
if (itemIDs == null) {
throw new IllegalArgumentException("itemIDs is null");
}
@@ -215,12 +220,12 @@
if (prefs1 == null || prefs2 == null) {
return 0;
}
- Set<Comparable<?>> users1 = new FastSet<Comparable<?>>(prefs1.length());
+ FastIDSet users1 = new FastIDSet(prefs1.length());
int size1 = prefs1.length();
for (int i = 0; i < size1; i++) {
users1.add(prefs1.getUserID(i));
}
- Set<Comparable<?>> users2 = new FastSet<Comparable<?>>(prefs2.length());
+ FastIDSet users2 = new FastIDSet(prefs2.length());
int size2 = prefs2.length();
for (int i = 0; i < size2; i++) {
users2.add(prefs2.getUserID(i));
@@ -231,12 +236,12 @@
}
@Override
- public void removePreference(Comparable<?> userID, Comparable<?> itemID) throws NoSuchUserException {
+ public void removePreference(long userID, long itemID) throws NoSuchUserException {
throw new UnsupportedOperationException();
}
@Override
- public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value) throws NoSuchUserException {
+ public void setPreference(long userID, long itemID, float value) throws NoSuchUserException {
throw new UnsupportedOperationException();
}
@@ -247,7 +252,19 @@
@Override
public String toString() {
- return "GenericDataModel[users:" + (userIDs.size() > 3 ? userIDs.subList(0, 3) + "..." : userIDs) + ']';
+ StringBuilder result = new StringBuilder(200);
+ result.append("GenericDataModel[users:");
+ for (int i = 0; i < Math.min(3, userIDs.length); i++) {
+ if (i > 0) {
+ result.append(',');
+ }
+ result.append(userIDs[i]);
+ }
+ if (result.length() > 3) {
+ result.append("...");
+ }
+ result.append(']');
+ return result.toString();
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericItemPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -38,15 +38,15 @@
private static final int VALUE = 2;
private static final int VALUE_REVERSED = 3;
- private final Comparable<?>[] IDs;
- private Comparable<?> id;
+ private final long[] IDs;
+ private long id;
private final float[] values;
public GenericItemPreferenceArray(int size) {
if (size < 1) {
throw new IllegalArgumentException("size is less than 1");
}
- this.IDs = new Comparable<?>[size];
+ this.IDs = new long[size];
values = new float[size];
}
@@ -77,22 +77,22 @@
}
@Override
- public Comparable<?> getUserID(int i) {
+ public long getUserID(int i) {
return IDs[i];
}
@Override
- public void setUserID(int i, Comparable<?> userID) {
+ public void setUserID(int i, long userID) {
IDs[i] = userID;
}
@Override
- public Comparable<?> getItemID(int i) {
+ public long getItemID(int i) {
return id;
}
@Override
- public void setItemID(int i, Comparable<?> itemID) {
+ public void setItemID(int i, long itemID) {
id = itemID;
}
@@ -144,7 +144,7 @@
private boolean isLess(int i, int j, int type) {
switch (type) {
case USER:
- return ((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0;
+ return IDs[i] < IDs[j];
case VALUE:
return values[i] < values[j];
case VALUE_REVERSED:
@@ -155,7 +155,7 @@
}
private void swap(int i, int j) {
- Comparable<?> temp1 = IDs[i];
+ long temp1 = IDs[i];
float temp2 = values[i];
IDs[i] = IDs[j];
values[i] = values[j];
@@ -201,12 +201,12 @@
}
@Override
- public Comparable<?> getUserID() {
+ public long getUserID() {
return GenericItemPreferenceArray.this.getUserID(i);
}
@Override
- public Comparable<?> getItemID() {
+ public long getItemID() {
return GenericItemPreferenceArray.this.getItemID(i);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericPreference.java Tue Aug 11 12:04:35 2009
@@ -24,14 +24,11 @@
/** <p>A simple {@link Preference} encapsulating an item and preference value.</p> */
public class GenericPreference implements Preference, Serializable {
- private final Comparable<?> userID;
- private final Comparable<?> itemID;
+ private final long userID;
+ private final long itemID;
private float value;
- public GenericPreference(Comparable<?> userID, Comparable<?> itemID, float value) {
- if (userID == null || itemID == null) {
- throw new IllegalArgumentException("userID or itemID is null");
- }
+ public GenericPreference(long userID, long itemID, float value) {
if (Float.isNaN(value)) {
throw new IllegalArgumentException("Invalid value: " + value);
}
@@ -41,12 +38,12 @@
}
@Override
- public Comparable<?> getUserID() {
+ public long getUserID() {
return userID;
}
@Override
- public Comparable<?> getItemID() {
+ public long getItemID() {
return itemID;
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/GenericUserPreferenceArray.java Tue Aug 11 12:04:35 2009
@@ -39,15 +39,15 @@
private static final int VALUE = 2;
private static final int VALUE_REVERSED = 3;
- private final Comparable<?>[] IDs;
- private Comparable<?> id;
+ private final long[] IDs;
+ private long id;
private final float[] values;
public GenericUserPreferenceArray(int size) {
if (size < 1) {
throw new IllegalArgumentException("size is less than 1");
}
- this.IDs = new Comparable<?>[size];
+ this.IDs = new long[size];
values = new float[size];
}
@@ -78,22 +78,22 @@
}
@Override
- public Comparable<?> getUserID(int i) {
+ public long getUserID(int i) {
return id;
}
@Override
- public void setUserID(int i, Comparable<?> userID) {
+ public void setUserID(int i, long userID) {
id = userID;
}
@Override
- public Comparable<?> getItemID(int i) {
+ public long getItemID(int i) {
return IDs[i];
}
@Override
- public void setItemID(int i, Comparable<?> itemID) {
+ public void setItemID(int i, long itemID) {
IDs[i] = itemID;
}
@@ -145,7 +145,7 @@
private boolean isLess(int i, int j, int type) {
switch (type) {
case ITEM:
- return ((Comparable<Object>) IDs[i]).compareTo(IDs[j]) < 0;
+ return IDs[i] < IDs[j];
case VALUE:
return values[i] < values[j];
case VALUE_REVERSED:
@@ -156,7 +156,7 @@
}
private void swap(int i, int j) {
- Comparable<?> temp1 = IDs[i];
+ long temp1 = IDs[i];
float temp2 = values[i];
IDs[i] = IDs[j];
values[i] = values[j];
@@ -202,12 +202,12 @@
}
@Override
- public Comparable<?> getUserID() {
+ public long getUserID() {
return GenericUserPreferenceArray.this.getUserID(i);
}
@Override
- public Comparable<?> getItemID() {
+ public long getItemID() {
return GenericUserPreferenceArray.this.getItemID(i);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/file/FileDataModel.java Tue Aug 11 12:04:35 2009
@@ -19,9 +19,10 @@
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.FileLineIterator;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.model.GenericBooleanPrefDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericPreference;
@@ -75,8 +76,8 @@
* that, a JDBC-backed {@link DataModel} and a database are more appropriate.</p>
*
* <p>It is possible and likely useful to subclass this class and customize its behavior to accommodate
- * application-specific needs and input formats. See {@link #processLine(String, Map, char)} and
- * {@link #processLineWithoutID(String, Map, char)}
+ * application-specific needs and input formats. See {@link #processLine(String, FastByIDMap, char)} and
+ * {@link #processLineWithoutID(String, FastByIDMap, char)}
*/
public class FileDataModel implements DataModel {
@@ -138,17 +139,17 @@
FileLineIterator iterator = new FileLineIterator(dataFile, false);
String firstLine = iterator.peek();
char delimiter = determineDelimiter(firstLine);
- boolean hasPrefValues = firstLine.indexOf(',', firstLine.indexOf(',') + 1) >= 0;
+ boolean hasPrefValues = firstLine.indexOf(delimiter, firstLine.indexOf(delimiter) + 1) >= 0;
if (hasPrefValues) {
- Map<Comparable<?>, Collection<Preference>> data = new FastMap<Comparable<?>, Collection<Preference>>();
+ FastByIDMap<Collection<Preference>> data = new FastByIDMap<Collection<Preference>>();
processFile(iterator, data, delimiter);
for (File updateFile : findUpdateFiles()) {
processFile(new FileLineIterator(updateFile, false), data, delimiter);
}
- return new GenericDataModel(GenericDataModel.toPrefArrayValues(data, true));
+ return new GenericDataModel(GenericDataModel.toDataMap(data, true));
} else {
- Map<Comparable<?>, FastSet<Comparable<?>>> data = new FastMap<Comparable<?>, FastSet<Comparable<?>>>();
+ FastByIDMap<FastIDSet> data = new FastByIDMap<FastIDSet>();
processFileWithoutID(iterator, data, delimiter);
for (File updateFile : findUpdateFiles()) {
processFileWithoutID(new FileLineIterator(updateFile, false), data, delimiter);
@@ -189,7 +190,7 @@
}
protected void processFile(FileLineIterator dataOrUpdateFileIterator,
- Map<Comparable<?>, Collection<Preference>> data,
+ FastByIDMap<Collection<Preference>> data,
char delimiter) {
log.info("Reading file info...");
AtomicInteger count = new AtomicInteger();
@@ -217,7 +218,7 @@
* @param line line from input data file
* @param data all data read so far, as a mapping from user IDs to preferences
*/
- protected void processLine(String line, Map<Comparable<?>, Collection<Preference>> data, char delimiter) {
+ protected void processLine(String line, FastByIDMap<Collection<Preference>> data, char delimiter) {
if (line.length() == 0 || line.charAt(0) == '#') {
return;
@@ -229,12 +230,12 @@
throw new IllegalArgumentException("Bad line: " + line);
}
- String userID = line.substring(0, delimiterOne);
- String itemID = line.substring(delimiterOne + 1, delimiterTwo);
+ long userID = Long.parseLong(line.substring(0, delimiterOne));
+ long itemID = Long.parseLong(line.substring(delimiterOne + 1, delimiterTwo));
String preferenceValueString = line.substring(delimiterTwo + 1);
if (transpose) {
- String tmp = userID;
+ long tmp = userID;
userID = itemID;
itemID = tmp;
}
@@ -249,7 +250,7 @@
Iterator<Preference> prefsIterator = prefs.iterator();
while (prefsIterator.hasNext()) {
Preference pref = prefsIterator.next();
- if (pref.getItemID().equals(itemID)) {
+ if (pref.getItemID() == itemID) {
prefsIterator.remove();
break;
}
@@ -261,7 +262,7 @@
}
protected void processFileWithoutID(FileLineIterator dataOrUpdateFileIterator,
- Map<Comparable<?>, FastSet<Comparable<?>>> data,
+ FastByIDMap<FastIDSet> data,
char delimiter) {
log.info("Reading file info...");
AtomicInteger count = new AtomicInteger();
@@ -278,7 +279,7 @@
log.info("Read lines: {}", count.get());
}
- protected void processLineWithoutID(String line, Map<Comparable<?>, FastSet<Comparable<?>>> data, char delimiter) {
+ protected void processLineWithoutID(String line, FastByIDMap<FastIDSet> data, char delimiter) {
if (line.length() == 0 || line.charAt(0) == '#') {
return;
@@ -289,17 +290,17 @@
throw new IllegalArgumentException("Bad line: " + line);
}
- String userID = line.substring(0, delimiterOne);
- String itemID = line.substring(delimiterOne + 1);
+ long userID = Long.parseLong(line.substring(0, delimiterOne));
+ long itemID = Long.parseLong(line.substring(delimiterOne + 1));
if (transpose) {
- String tmp = userID;
+ long tmp = userID;
userID = itemID;
itemID = tmp;
}
- FastSet<Comparable<?>> itemIDs = data.get(userID);
+ FastIDSet itemIDs = data.get(userID);
if (itemIDs == null) {
- itemIDs = new FastSet<Comparable<?>>(2);
+ itemIDs = new FastIDSet(2);
data.put(userID, itemIDs);
}
itemIDs.add(itemID);
@@ -312,36 +313,36 @@
}
@Override
- public Iterable<Comparable<?>> getUserIDs() throws TasteException {
+ public LongPrimitiveIterator getUserIDs() throws TasteException {
checkLoaded();
return delegate.getUserIDs();
}
@Override
- public PreferenceArray getPreferencesFromUser(Comparable<?> userID) throws TasteException {
+ public PreferenceArray getPreferencesFromUser(long userID) throws TasteException {
checkLoaded();
return delegate.getPreferencesFromUser(userID);
}
@Override
- public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> userID) throws TasteException {
+ public FastIDSet getItemIDsFromUser(long userID) throws TasteException {
return delegate.getItemIDsFromUser(userID);
}
@Override
- public Iterable<Comparable<?>> getItemIDs() throws TasteException {
+ public LongPrimitiveIterator getItemIDs() throws TasteException {
checkLoaded();
return delegate.getItemIDs();
}
@Override
- public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws TasteException {
+ public PreferenceArray getPreferencesForItem(long itemID) throws TasteException {
checkLoaded();
return delegate.getPreferencesForItem(itemID);
}
@Override
- public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+ public Float getPreferenceValue(long userID, long itemID) throws TasteException {
return delegate.getPreferenceValue(userID, itemID);
}
@@ -358,7 +359,7 @@
}
@Override
- public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) throws TasteException {
+ public int getNumUsersWithPreferenceFor(long... itemIDs) throws TasteException {
checkLoaded();
return delegate.getNumUsersWithPreferenceFor(itemIDs);
}
@@ -369,14 +370,14 @@
* reloaded from a file. This method should also be considered relatively slow.
*/
@Override
- public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value) throws TasteException {
+ public void setPreference(long userID, long itemID, float value) throws TasteException {
checkLoaded();
delegate.setPreference(userID, itemID, value);
}
- /** See the warning at {@link #setPreference(Comparable, Comparable, float)}. */
+ /** See the warning at {@link #setPreference(long, long, float)}. */
@Override
- public void removePreference(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+ public void removePreference(long userID, long itemID) throws TasteException {
checkLoaded();
delegate.removePreference(userID, itemID);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -75,15 +75,11 @@
@Override
protected Preference buildPreference(ResultSet rs) throws SQLException {
- return new BooleanPreference((Comparable<?>) rs.getObject(1), (Comparable<?>) rs.getObject(2));
+ return new BooleanPreference(rs.getLong(1), rs.getLong(2));
}
@Override
- public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value)
- throws TasteException {
- if (userID == null || itemID == null) {
- throw new IllegalArgumentException("userID or itemID is null");
- }
+ public void setPreference(long userID, long itemID, float value) throws TasteException {
if (!Float.isNaN(value)) {
throw new IllegalArgumentException("Invalid value: " + value);
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -21,10 +21,10 @@
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.Cache;
-import org.apache.mahout.cf.taste.impl.common.FastMap;
-import org.apache.mahout.cf.taste.impl.common.FastSet;
+import org.apache.mahout.cf.taste.impl.common.FastByIDMap;
+import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.IOUtils;
-import org.apache.mahout.cf.taste.impl.common.IteratorIterable;
+import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
import org.apache.mahout.cf.taste.impl.common.Retriever;
import org.apache.mahout.cf.taste.impl.common.SkippingIterator;
import org.apache.mahout.cf.taste.impl.common.jdbc.AbstractJDBCComponent;
@@ -47,7 +47,6 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
-import java.util.Map;
import java.util.NoSuchElementException;
/**
@@ -91,7 +90,7 @@
private final String getNumPreferenceForItemsSQL;
private int cachedNumUsers;
private int cachedNumItems;
- private final Cache<Comparable<?>, Integer> itemPrefCounts;
+ private final Cache<Long, Integer> itemPrefCounts;
protected AbstractJDBCDataModel(DataSource dataSource,
String getPreferenceSQL,
@@ -190,7 +189,7 @@
this.cachedNumUsers = -1;
this.cachedNumItems = -1;
- this.itemPrefCounts = new Cache<Comparable<?>, Integer>(new ItemPrefCountRetriever(getNumPreferenceForItemSQL));
+ this.itemPrefCounts = new Cache<Long, Integer>(new ItemPrefCountRetriever(getNumPreferenceForItemSQL));
}
@@ -217,14 +216,14 @@
}
@Override
- public Iterable<Comparable<?>> getUserIDs() throws TasteException {
+ public LongPrimitiveIterator getUserIDs() throws TasteException {
log.debug("Retrieving all users...");
- return new IteratorIterable<Comparable<?>>(new ResultSetIDIterator(getUsersSQL));
+ return new ResultSetIDIterator(getUsersSQL);
}
/** @throws NoSuchUserException if there is no such user */
@Override
- public PreferenceArray getPreferencesFromUser(Comparable<?> id) throws TasteException {
+ public PreferenceArray getPreferencesFromUser(long id) throws TasteException {
log.debug("Retrieving user ID '{}'", id);
@@ -237,7 +236,7 @@
stmt = conn.prepareStatement(getUserSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
stmt.setFetchSize(getFetchSize());
- stmt.setObject(1, id);
+ stmt.setLong(1, id);
log.debug("Executing SQL query: {}", getUserSQL);
rs = stmt.executeQuery();
@@ -263,14 +262,14 @@
}
@Override
- public Map<Comparable<?>, PreferenceArray> exportWithPrefs() throws TasteException {
+ public FastByIDMap<PreferenceArray> exportWithPrefs() throws TasteException {
log.debug("Exporting all data");
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
- Map<Comparable<?>, PreferenceArray> result = new FastMap<Comparable<?>, PreferenceArray>();
+ FastByIDMap<PreferenceArray> result = new FastByIDMap<PreferenceArray>();
try {
conn = dataSource.getConnection();
@@ -281,10 +280,10 @@
log.debug("Executing SQL query: {}", getAllUsersSQL);
rs = stmt.executeQuery(getAllUsersSQL);
- Comparable<?> currentUserID = null;
+ Long currentUserID = null;
List<Preference> currentPrefs = new ArrayList<Preference>();
while (rs.next()) {
- Comparable<?> nextUserID = (Comparable<?>) rs.getObject(1);
+ long nextUserID = rs.getLong(1);
if (currentUserID != null && !currentUserID.equals(nextUserID)) {
if (!currentPrefs.isEmpty()) {
result.put(currentUserID, new GenericUserPreferenceArray(currentPrefs));
@@ -311,14 +310,14 @@
}
@Override
- public Map<Comparable<?>, FastSet<Comparable<?>>> exportWithIDsOnly() throws TasteException {
+ public FastByIDMap<FastIDSet> exportWithIDsOnly() throws TasteException {
log.debug("Exporting all data");
Connection conn = null;
Statement stmt = null;
ResultSet rs = null;
- Map<Comparable<?>, FastSet<Comparable<?>>> result = new FastMap<Comparable<?>, FastSet<Comparable<?>>>();
+ FastByIDMap<FastIDSet> result = new FastByIDMap<FastIDSet>();
try {
conn = dataSource.getConnection();
@@ -329,19 +328,21 @@
log.debug("Executing SQL query: {}", getAllUsersSQL);
rs = stmt.executeQuery(getAllUsersSQL);
- Comparable<?> currentUserID = null;
- FastSet<Comparable<?>> currentItemIDs = new FastSet<Comparable<?>>(2);
+ boolean currentUserIDSet = false;
+ long currentUserID = 0L; // value isn't used
+ FastIDSet currentItemIDs = new FastIDSet(2);
while (rs.next()) {
- Comparable<?> nextUserID = (Comparable<?>) rs.getObject(1);
- if (currentUserID != null && !currentUserID.equals(nextUserID)) {
+ long nextUserID = rs.getLong(1);
+ if (currentUserIDSet && currentUserID != nextUserID) {
if (!currentItemIDs.isEmpty()) {
result.put(currentUserID, currentItemIDs);
- currentItemIDs = new FastSet<Comparable<?>>(2);
+ currentItemIDs = new FastIDSet(2);
}
} else {
- currentItemIDs.add((Comparable<?>) rs.getObject(2));
+ currentItemIDs.add(rs.getLong(2));
}
currentUserID = nextUserID;
+ currentUserIDSet = true;
}
if (!currentItemIDs.isEmpty()) {
result.put(currentUserID, currentItemIDs);
@@ -360,7 +361,7 @@
/** @throws NoSuchUserException if there is no such user */
@Override
- public FastSet<Comparable<?>> getItemIDsFromUser(Comparable<?> id) throws TasteException {
+ public FastIDSet getItemIDsFromUser(long id) throws TasteException {
log.debug("Retrieving items for user ID '{}'", id);
@@ -373,14 +374,14 @@
stmt = conn.prepareStatement(getUserSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
stmt.setFetchSize(getFetchSize());
- stmt.setObject(1, id);
+ stmt.setLong(1, id);
log.debug("Executing SQL query: {}", getUserSQL);
rs = stmt.executeQuery();
- FastSet<Comparable<?>> result = new FastSet<Comparable<?>>();
+ FastIDSet result = new FastIDSet();
while (rs.next()) {
- result.add((Comparable<?>) rs.getObject(1));
+ result.add(rs.getLong(1));
}
if (result.isEmpty()) {
@@ -399,7 +400,7 @@
}
@Override
- public Float getPreferenceValue(Comparable<?> userID, Comparable<?> itemID) throws TasteException {
+ public Float getPreferenceValue(long userID, long itemID) throws TasteException {
log.debug("Retrieving preferences for item ID '{}'", itemID);
Connection conn = null;
PreparedStatement stmt = null;
@@ -409,8 +410,8 @@
stmt = conn.prepareStatement(getPreferenceSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
stmt.setFetchSize(1);
- stmt.setObject(1, userID);
- stmt.setObject(2, itemID);
+ stmt.setLong(1, userID);
+ stmt.setLong(2, itemID);
log.debug("Executing SQL query: {}", getPreferenceSQL);
rs = stmt.executeQuery();
@@ -428,18 +429,18 @@
}
@Override
- public Iterable<Comparable<?>> getItemIDs() throws TasteException {
+ public LongPrimitiveIterator getItemIDs() throws TasteException {
log.debug("Retrieving all items...");
- return new IteratorIterable<Comparable<?>>(new ResultSetIDIterator(getItemsSQL));
+ return new ResultSetIDIterator(getItemsSQL);
}
@Override
- public PreferenceArray getPreferencesForItem(Comparable<?> itemID) throws TasteException {
+ public PreferenceArray getPreferencesForItem(long itemID) throws TasteException {
List<Preference> list = doGetPreferencesForItem(itemID);
return new GenericItemPreferenceArray(list);
}
- protected List<Preference> doGetPreferencesForItem(Comparable<?> itemID) throws TasteException {
+ protected List<Preference> doGetPreferencesForItem(long itemID) throws TasteException {
log.debug("Retrieving preferences for item ID '{}'", itemID);
Connection conn = null;
PreparedStatement stmt = null;
@@ -449,7 +450,7 @@
stmt = conn.prepareStatement(getPrefsForItemSQL, ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
stmt.setFetchDirection(ResultSet.FETCH_FORWARD);
stmt.setFetchSize(getFetchSize());
- stmt.setObject(1, itemID);
+ stmt.setLong(1, itemID);
log.debug("Executing SQL query: {}", getPrefsForItemSQL);
rs = stmt.executeQuery();
@@ -483,7 +484,7 @@
}
@Override
- public int getNumUsersWithPreferenceFor(Comparable<?>... itemIDs) throws TasteException {
+ public int getNumUsersWithPreferenceFor(long... itemIDs) throws TasteException {
if (itemIDs == null) {
throw new IllegalArgumentException("itemIDs is null");
}
@@ -493,11 +494,11 @@
}
return length == 1 ?
itemPrefCounts.get(itemIDs[0]) :
- getNumThings("user preferring items", getNumPreferenceForItemsSQL, (Object[]) itemIDs);
+ getNumThings("user preferring items", getNumPreferenceForItemsSQL, itemIDs);
}
- private int getNumThings(String name, String sql, Object... args) throws TasteException {
+ private int getNumThings(String name, String sql, long... args) throws TasteException {
log.debug("Retrieving number of {} in model", name);
Connection conn = null;
PreparedStatement stmt = null;
@@ -509,7 +510,7 @@
stmt.setFetchSize(getFetchSize());
if (args != null) {
for (int i = 1; i <= args.length; i++) {
- stmt.setObject(i, args[i - 1]);
+ stmt.setLong(i, args[i - 1]);
}
}
log.debug("Executing SQL query: {}", sql);
@@ -525,11 +526,7 @@
}
@Override
- public void setPreference(Comparable<?> userID, Comparable<?> itemID, float value)
- throws TasteException {
- if (userID == null || itemID == null) {
- throw new IllegalArgumentException("userID or itemID is null");
- }
+ public void setPreference(long userID, long itemID, float value) throws TasteException {
if (Float.isNaN(value)) {
throw new IllegalArgumentException("Invalid value: " + value);
}
@@ -542,8 +539,8 @@
try {
conn = dataSource.getConnection();
stmt = conn.prepareStatement(setPreferenceSQL);
- stmt.setObject(1, userID);
- stmt.setObject(2, itemID);
+ stmt.setLong(1, userID);
+ stmt.setLong(2, itemID);
stmt.setDouble(3, value);
stmt.setDouble(4, value);
@@ -559,11 +556,7 @@
}
@Override
- public void removePreference(Comparable<?> userID, Comparable<?> itemID)
- throws TasteException {
- if (userID == null || itemID == null) {
- throw new IllegalArgumentException("userID or itemID is null");
- }
+ public void removePreference(long userID, long itemID) throws TasteException {
log.debug("Removing preference for user '{}', item '{}'", userID, itemID);
@@ -573,8 +566,8 @@
try {
conn = dataSource.getConnection();
stmt = conn.prepareStatement(removePreferenceSQL);
- stmt.setObject(1, userID);
- stmt.setObject(2, itemID);
+ stmt.setLong(1, userID);
+ stmt.setLong(2, itemID);
log.debug("Executing SQL update: {}", removePreferenceSQL);
stmt.executeUpdate();
@@ -595,7 +588,7 @@
}
protected Preference buildPreference(ResultSet rs) throws SQLException {
- return new GenericPreference((Comparable<?>) rs.getObject(1), (Comparable<?>) rs.getObject(2), rs.getFloat(3));
+ return new GenericPreference(rs.getLong(1), rs.getLong(2), rs.getFloat(3));
}
/**
@@ -605,7 +598,7 @@
* only release database resources after {@link #hasNext()} has been called and has returned <code>false</code>;
* callers should make sure to "drain" the entire set of data to avoid tying up database resources.</p>
*/
- private final class ResultSetIDIterator implements SkippingIterator<Comparable<?>> {
+ private final class ResultSetIDIterator implements LongPrimitiveIterator, SkippingIterator<Long> {
private final Connection connection;
private final Statement statement;
@@ -648,15 +641,19 @@
return nextExists;
}
+ public Long next() {
+ return nextLong();
+ }
+
@Override
- public Comparable<?> next() {
+ public long nextLong() {
if (!hasNext()) {
throw new NoSuchElementException();
}
try {
- Comparable<?> ID = (Comparable<?>) resultSet.getObject(1);
+ long ID = resultSet.getLong(1);
resultSet.next();
return ID;
} catch (SQLException sqle) {
@@ -668,6 +665,22 @@
}
+ @Override
+ public long peek() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ try {
+ return resultSet.getLong(1);
+ } catch (SQLException sqle) {
+ // No good way to handle this since we can't throw an exception
+ log.warn("Exception while iterating", sqle);
+ close();
+ throw new NoSuchElementException("Can't retrieve more due to exception: " + sqle);
+ }
+
+ }
+
/**
* @throws UnsupportedOperationException
*/
@@ -695,7 +708,7 @@
}
- private class ItemPrefCountRetriever implements Retriever<Comparable<?>, Integer> {
+ private class ItemPrefCountRetriever implements Retriever<Long, Integer> {
private final String getNumPreferenceForItemSQL;
private ItemPrefCountRetriever(String getNumPreferenceForItemSQL) {
@@ -703,7 +716,7 @@
}
@Override
- public Integer get(Comparable<?> key) throws TasteException {
+ public Integer get(Long key) throws TasteException {
return getNumThings("user preferring item", getNumPreferenceForItemSQL, key);
}
}
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -28,8 +28,8 @@
*
* <pre>
* CREATE TABLE taste_preferences (
- * user_id INT NOT NULL,
- * item_id INT NOT NULL,
+ * user_id BIGINT NOT NULL,
+ * item_id BIGINT NOT NULL,
* PRIMARY KEY (user_id, item_id),
* INDEX (user_id),
* INDEX (item_id)
@@ -122,7 +122,7 @@
"SELECT COUNT(1) FROM " + preferenceTable + " WHERE " + itemIDColumn + "=?",
// getNumPreferenceForItemsSQL
"SELECT COUNT(1) FROM " + preferenceTable + " tp1 JOIN " + preferenceTable + " tp2 " +
- "USING " + userIDColumn + " WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
+ "USING (" + userIDColumn + ") WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
}
@Override
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java?rev=803081&r1=803080&r2=803081&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java Tue Aug 11 12:04:35 2009
@@ -31,21 +31,21 @@
*
* <table>
* <tr><th>user_id</th><th>item_id</th><th>preference</th></tr>
- * <tr><td>ABC</td><td>123</td><td>0.9</td></tr>
- * <tr><td>ABC</td><td>456</td><td>0.1</td></tr>
- * <tr><td>DEF</td><td>123</td><td>0.2</td></tr>
- * <tr><td>DEF</td><td>789</td><td>0.3</td></tr>
+ * <tr><td>987</td><td>123</td><td>0.9</td></tr>
+ * <tr><td>987</td><td>456</td><td>0.1</td></tr>
+ * <tr><td>654</td><td>123</td><td>0.2</td></tr>
+ * <tr><td>654</td><td>789</td><td>0.3</td></tr>
* </table>
*
* <p><code>preference</code> must have a type compatible
- * with the Java <code>float</code> type. <code>user_id</code> and <code>item_id</code> should be an integer or
- * string type (INT, LONGINT, VARCHAR). For example, the following command sets up a suitable table in MySQL,
+ * with the Java <code>float</code> type. <code>user_id</code> and <code>item_id</code> should be compatible
+ * with long type (BIGINT). For example, the following command sets up a suitable table in MySQL,
* complete with primary key and indexes:</p>
*
* <pre>
* CREATE TABLE taste_preferences (
- * user_id INT NOT NULL,
- * item_id INT NOT NULL,
+ * user_id BIGINT NOT NULL,
+ * item_id BIGINT NOT NULL,
* preference FLOAT NOT NULL,
* PRIMARY KEY (user_id, item_id),
* INDEX (user_id),
@@ -174,7 +174,7 @@
"SELECT COUNT(1) FROM " + preferenceTable + " WHERE " + itemIDColumn + "=?",
// getNumPreferenceForItemsSQL
"SELECT COUNT(1) FROM " + preferenceTable + " tp1 JOIN " + preferenceTable + " tp2 " +
- "USING " + userIDColumn + " WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
+ "USING (" + userIDColumn + ") WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
}
@Override