You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2009/07/07 18:45:26 UTC
svn commit: r791905 - in
/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl:
eval/ model/jdbc/
Author: srowen
Date: Tue Jul 7 16:45:25 2009
New Revision: 791905
URL: http://svn.apache.org/viewvc?rev=791905&view=rev
Log:
Add initial support for evaluating 'boolean'-based setups
Added:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java
- copied, changed from r791748, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java
- copied, changed from r791748, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java
Modified:
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java
lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java?rev=791905&r1=791904&r2=791905&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/eval/GenericRecommenderIRStatsEvaluator.java Tue Jul 7 16:45:25 2009
@@ -30,6 +30,8 @@
import org.apache.mahout.cf.taste.impl.common.FastSet;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericUser;
+import org.apache.mahout.cf.taste.impl.model.ByValuePreferenceComparator;
+import org.apache.mahout.cf.taste.impl.model.BooleanPrefUser;
import org.apache.mahout.cf.taste.model.DataModel;
import org.apache.mahout.cf.taste.model.Item;
import org.apache.mahout.cf.taste.model.Preference;
@@ -44,6 +46,8 @@
import java.util.Collection;
import java.util.List;
import java.util.Random;
+import java.util.Collections;
+import java.util.Arrays;
/**
* <p>For each {@link User}, these implementation determine the top <code>n</code> preferences,
@@ -102,8 +106,16 @@
Object id = user.getID();
Collection<Item> relevantItems = new FastSet<Item>(at);
Preference[] prefs = user.getPreferencesAsArray();
+ if (prefs.length < 2 * at) {
+ // Really not enough prefs to meaningfully evaluate this user
+ continue;
+ }
+
+ // List some most-preferred items that would count as (most) "relevant" results
double theRelevanceThreshold = Double.isNaN(relevanceThreshold) ? computeThreshold(prefs) : relevanceThreshold;
- for (Preference pref : prefs) {
+ Arrays.sort(prefs, Collections.reverseOrder(ByValuePreferenceComparator.getInstance()));
+ for (int i = 0; i < prefs.length && relevantItems.size() < at; i++) {
+ Preference pref = prefs[i];
if (pref.getValue() >= theRelevanceThreshold) {
relevantItems.add(pref.getItem());
}
@@ -163,7 +175,17 @@
}
}
if (!trainingPrefs.isEmpty()) {
- User trainingUser = new GenericUser<String>(id.toString(), trainingPrefs);
+ // TODO hack
+ User trainingUser;
+ if (user2 instanceof BooleanPrefUser) {
+ FastSet<Object> itemIDs = new FastSet<Object>();
+ for (Preference pref : trainingPrefs) {
+ itemIDs.add(pref.getItem().getID());
+ }
+ trainingUser = new BooleanPrefUser<String>(id.toString(), itemIDs);
+ } else {
+ trainingUser = new GenericUser<String>(id.toString(), trainingPrefs);
+ }
trainingUsers.add(trainingUser);
}
} else {
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java (from r791748, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java&r1=791748&r2=791905&rev=791905&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractBooleanPrefJDBCDataModel.java Tue Jul 7 16:45:25 2009
@@ -17,240 +17,75 @@
package org.apache.mahout.cf.taste.impl.model.jdbc;
-import org.apache.mahout.cf.taste.common.Refreshable;
+import org.apache.mahout.cf.taste.model.User;
+import org.apache.mahout.cf.taste.model.Preference;
+import org.apache.mahout.cf.taste.model.Item;
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.common.NoSuchUserException;
import org.apache.mahout.cf.taste.impl.common.IOUtils;
+import org.apache.mahout.cf.taste.impl.common.FastSet;
import org.apache.mahout.cf.taste.impl.common.IteratorIterable;
-import org.apache.mahout.cf.taste.impl.model.GenericItem;
-import org.apache.mahout.cf.taste.impl.model.GenericPreference;
-import org.apache.mahout.cf.taste.impl.model.GenericUser;
-import org.apache.mahout.cf.taste.model.DataModel;
-import org.apache.mahout.cf.taste.model.Item;
-import org.apache.mahout.cf.taste.model.JDBCDataModel;
-import org.apache.mahout.cf.taste.model.Preference;
-import org.apache.mahout.cf.taste.model.User;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
+import org.apache.mahout.cf.taste.impl.model.BooleanPrefUser;
+import org.apache.mahout.cf.taste.impl.model.BooleanPreference;
-import javax.naming.Context;
-import javax.naming.InitialContext;
-import javax.naming.NamingException;
import javax.sql.DataSource;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.List;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Iterator;
-import java.util.List;
import java.util.NoSuchElementException;
-/**
- * <p>An abstract superclass for JDBC-related {@link DataModel} implementations, providing most of the common
- * functionality that any such implementation would need.</p>
- *
- * <p>Performance will be a concern with any JDBC-based {@link DataModel}. There are going to be lots of
- * simultaneous reads and some writes to one table. Make sure the table is set up optimally -- for example,
- * you'll want to establish indexes.</p>
- *
- * <p>You'll also want to use connection pooling of some kind. Most J2EE containers like Tomcat
- * provide connection pooling, so make sure the {@link DataSource} it exposes is using pooling. Outside a
- * J2EE container, you can use packages like Jakarta's
- * <a href="http://jakarta.apache.org/commons/dbcp/">DBCP</a> to create a {@link DataSource} on top of your
- * database whose {@link Connection}s are pooled.</p>
- *
- * <p>Also note: this default implementation assumes that the user and item ID keys are {@link String}s, for
- * maximum flexibility. You can override this behavior by subclassing an implementation and overriding
- * {@link #buildItem(String)} and {@link #buildUser(String, List)}. If you don't, just make sure you use
- * {@link String}s as IDs throughout your code. If your IDs are really numeric, and you use, say, {@link Long}
- * for IDs in the rest of your code, you will run into subtle problems because the {@link Long} values won't
- * be equal to or compare correctly to the underlying {@link String} key values.</p>
- */
-public abstract class AbstractJDBCDataModel implements JDBCDataModel {
-
- static final Logger log = LoggerFactory.getLogger(AbstractJDBCDataModel.class);
-
- public static final String DEFAULT_DATASOURCE_NAME = "jdbc/taste";
- public static final String DEFAULT_PREFERENCE_TABLE = "taste_preferences";
- public static final String DEFAULT_USER_ID_COLUMN = "user_id";
- public static final String DEFAULT_ITEM_ID_COLUMN = "item_id";
- public static final String DEFAULT_PREFERENCE_COLUMN = "preference";
-
- private final DataSource dataSource;
- private final String preferenceTable;
- private final String userIDColumn;
- private final String itemIDColumn;
- private final String preferenceColumn;
+
+public abstract class AbstractBooleanPrefJDBCDataModel extends AbstractJDBCDataModel {
+
private final String getUserSQL;
- private final String getNumItemsSQL;
- private final String getNumUsersSQL;
private final String setPreferenceSQL;
- private final String removePreferenceSQL;
private final String getUsersSQL;
- private final String getItemsSQL;
- private final String getItemSQL;
private final String getPrefsForItemSQL;
- private final String getNumPreferenceForItemSQL;
- private final String getNumPreferenceForItemsSQL;
-
- protected AbstractJDBCDataModel(DataSource dataSource,
- String getUserSQL,
- String getNumItemsSQL,
- String getNumUsersSQL,
- String setPreferenceSQL,
- String removePreferenceSQL,
- String getUsersSQL,
- String getItemsSQL,
- String getItemSQL,
- String getPrefsForItemSQL,
- String getNumPreferenceForItemSQL,
- String getNumPreferenceForItemsSQL) {
- this(dataSource,
- DEFAULT_PREFERENCE_TABLE,
- DEFAULT_USER_ID_COLUMN,
- DEFAULT_ITEM_ID_COLUMN,
- DEFAULT_PREFERENCE_COLUMN,
- getUserSQL,
- getNumItemsSQL,
- getNumUsersSQL,
- setPreferenceSQL,
- removePreferenceSQL,
- getUsersSQL,
- getItemsSQL,
- getItemSQL,
- getPrefsForItemSQL,
- getNumPreferenceForItemSQL,
- getNumPreferenceForItemsSQL);
- }
- protected AbstractJDBCDataModel(DataSource dataSource,
- String preferenceTable,
- String userIDColumn,
- String itemIDColumn,
- String preferenceColumn,
- String getUserSQL,
- String getNumItemsSQL,
- String getNumUsersSQL,
- String setPreferenceSQL,
- String removePreferenceSQL,
- String getUsersSQL,
- String getItemsSQL,
- String getItemSQL,
- String getPrefsForItemSQL,
- String getNumPreferenceForItemSQL,
- String getNumPreferenceForItemsSQL) {
-
- log.debug("Creating AbstractJDBCModel...");
-
- checkNotNullAndLog("preferenceTable", preferenceTable);
- checkNotNullAndLog("userIDColumn", userIDColumn);
- checkNotNullAndLog("itemIDColumn", itemIDColumn);
- checkNotNullAndLog("preferenceColumn", preferenceColumn);
-
- checkNotNullAndLog("dataSource", dataSource);
- checkNotNullAndLog("getUserSQL", getUserSQL);
- checkNotNullAndLog("getNumItemsSQL", getNumItemsSQL);
- checkNotNullAndLog("getNumUsersSQL", getNumUsersSQL);
- checkNotNullAndLog("setPreferenceSQL", setPreferenceSQL);
- checkNotNullAndLog("removePreferenceSQL", removePreferenceSQL);
- checkNotNullAndLog("getUsersSQL", getUsersSQL);
- checkNotNullAndLog("getItemsSQL", getItemsSQL);
- checkNotNullAndLog("getItemSQL", getItemSQL);
- checkNotNullAndLog("getPrefsForItemSQL", getPrefsForItemSQL);
- checkNotNullAndLog("getNumPreferenceForItemSQL", getNumPreferenceForItemSQL);
- checkNotNullAndLog("getNumPreferenceForItemsSQL", getNumPreferenceForItemsSQL);
-
- if (!(dataSource instanceof ConnectionPoolDataSource)) {
- log.warn("You are not using ConnectionPoolDataSource. Make sure your DataSource pools connections " +
- "to the database itself, or database performance will be severely reduced.");
- }
-
- this.preferenceTable = preferenceTable;
- this.userIDColumn = userIDColumn;
- this.itemIDColumn = itemIDColumn;
- this.preferenceColumn = preferenceColumn;
-
- this.dataSource = dataSource;
+ protected AbstractBooleanPrefJDBCDataModel(DataSource dataSource,
+ String preferenceTable,
+ String userIDColumn,
+ String itemIDColumn,
+ String preferenceColumn,
+ String getUserSQL,
+ String getNumItemsSQL,
+ String getNumUsersSQL,
+ String setPreferenceSQL,
+ String removePreferenceSQL,
+ String getUsersSQL,
+ String getItemsSQL,
+ String getItemSQL,
+ String getPrefsForItemSQL,
+ String getNumPreferenceForItemSQL,
+ String getNumPreferenceForItemsSQL) {
+ super(dataSource,
+ preferenceTable,
+ userIDColumn,
+ itemIDColumn,
+ preferenceColumn,
+ getUserSQL,
+ getNumItemsSQL,
+ getNumUsersSQL,
+ setPreferenceSQL,
+ removePreferenceSQL,
+ getUsersSQL,
+ getItemsSQL,
+ getItemSQL,
+ getPrefsForItemSQL,
+ getNumPreferenceForItemSQL,
+ getNumPreferenceForItemsSQL);
this.getUserSQL = getUserSQL;
- this.getNumItemsSQL = getNumItemsSQL;
- this.getNumUsersSQL = getNumUsersSQL;
this.setPreferenceSQL = setPreferenceSQL;
- this.removePreferenceSQL = removePreferenceSQL;
this.getUsersSQL = getUsersSQL;
- this.getItemsSQL = getItemsSQL;
- this.getItemSQL = getItemSQL;
this.getPrefsForItemSQL = getPrefsForItemSQL;
- this.getNumPreferenceForItemSQL = getNumPreferenceForItemSQL;
- this.getNumPreferenceForItemsSQL = getNumPreferenceForItemsSQL;
- }
-
- private static void checkNotNullAndLog(String argName, Object value) {
- if (value == null || value.toString().length() == 0) {
- throw new IllegalArgumentException(argName + " is null or empty");
- }
- log.debug("{}: {}", argName, value);
- }
-
- /**
- * <p>Looks up a {@link DataSource} by name from JNDI. "java:comp/env/" is prepended to the argument
- * before looking up the name in JNDI.</p>
- *
- * @param dataSourceName JNDI name where a {@link DataSource} is bound (e.g. "jdbc/taste")
- * @return {@link DataSource} under that JNDI name
- * @throws TasteException if a JNDI error occurs
- */
- public static DataSource lookupDataSource(String dataSourceName) throws TasteException {
- Context context = null;
- try {
- context = new InitialContext();
- return (DataSource) context.lookup("java:comp/env/" + dataSourceName);
- } catch (NamingException ne) {
- throw new TasteException(ne);
- } finally {
- if (context != null) {
- try {
- context.close();
- } catch (NamingException ne) {
- log.warn("Error while closing Context; continuing...", ne);
- }
- }
- }
- }
-
- /**
- * @return the {@link DataSource} that this instance is using
- */
- @Override
- public DataSource getDataSource() {
- return dataSource;
- }
-
- public String getPreferenceTable() {
- return preferenceTable;
- }
-
- public String getUserIDColumn() {
- return userIDColumn;
- }
-
- public String getItemIDColumn() {
- return itemIDColumn;
- }
-
- public String getPreferenceColumn() {
- return preferenceColumn;
- }
-
- @Override
- public Iterable<? extends User> getUsers() throws TasteException {
- log.debug("Retrieving all users...");
- return new IteratorIterable<User>(new ResultSetUserIterator(dataSource, getUsersSQL));
}
/**
- * @throws NoSuchUserException if there is no such user
+ * @throws org.apache.mahout.cf.taste.common.NoSuchUserException if there is no such user
*/
@Override
public User getUser(Object id) throws TasteException {
@@ -264,23 +99,23 @@
String idString = id.toString();
try {
- conn = dataSource.getConnection();
+ conn = getDataSource().getConnection();
stmt = conn.prepareStatement(getUserSQL);
stmt.setObject(1, id);
log.debug("Executing SQL query: {}", getUserSQL);
rs = stmt.executeQuery();
- List<Preference> prefs = new ArrayList<Preference>();
+ FastSet<Object> itemIDs = new FastSet<Object>();
while (rs.next()) {
- addPreference(rs, prefs);
+ itemIDs.add(rs.getObject(1));
}
- if (prefs.isEmpty()) {
+ if (itemIDs.isEmpty()) {
throw new NoSuchUserException();
}
- return buildUser(idString, prefs);
+ return buildUser(idString, itemIDs);
} catch (SQLException sqle) {
log.warn("Exception while retrieving user", sqle);
@@ -292,137 +127,9 @@
}
@Override
- public Iterable<? extends Item> getItems() throws TasteException {
- log.debug("Retrieving all items...");
- return new IteratorIterable<Item>(new ResultSetItemIterator(dataSource, getItemsSQL));
- }
-
- @Override
- public Item getItem(Object id) throws TasteException {
- return getItem(id, false);
- }
-
- @Override
- public Item getItem(Object id, boolean assumeExists) throws TasteException {
-
- if (assumeExists) {
- return buildItem((String) id);
- }
-
- log.debug("Retrieving item ID '{}'", id);
-
- Connection conn = null;
- PreparedStatement stmt = null;
- ResultSet rs = null;
-
- try {
- conn = dataSource.getConnection();
- stmt = conn.prepareStatement(getItemSQL);
- stmt.setObject(1, id);
-
- log.debug("Executing SQL query: {}", getItemSQL);
- rs = stmt.executeQuery();
- if (rs.next()) {
- return buildItem((String) id);
- } else {
- throw new NoSuchElementException();
- }
- } catch (SQLException sqle) {
- log.warn("Exception while retrieving item", sqle);
- throw new TasteException(sqle);
- } finally {
- IOUtils.quietClose(rs, stmt, conn);
- }
- }
-
- @Override
- public Iterable<? extends Preference> getPreferencesForItem(Object itemID) throws TasteException {
- return doGetPreferencesForItem(itemID);
- }
-
- @Override
- public Preference[] getPreferencesForItemAsArray(Object itemID) throws TasteException {
- List<? extends Preference> list = doGetPreferencesForItem(itemID);
- return list.toArray(new Preference[list.size()]);
- }
-
- private List<? extends Preference> doGetPreferencesForItem(Object itemID) throws TasteException {
- log.debug("Retrieving preferences for item ID '{}'", itemID);
- Item item = getItem(itemID, true);
- Connection conn = null;
- PreparedStatement stmt = null;
- ResultSet rs = null;
- try {
- conn = dataSource.getConnection();
- stmt = conn.prepareStatement(getPrefsForItemSQL);
- stmt.setObject(1, itemID);
-
- log.debug("Executing SQL query: {}", getPrefsForItemSQL);
- rs = stmt.executeQuery();
- List<Preference> prefs = new ArrayList<Preference>();
- while (rs.next()) {
- double preference = rs.getDouble(1);
- String userID = rs.getString(2);
- Preference pref = buildPreference(buildUser(userID, null), item, preference);
- prefs.add(pref);
- }
- return prefs;
- } catch (SQLException sqle) {
- log.warn("Exception while retrieving prefs for item", sqle);
- throw new TasteException(sqle);
- } finally {
- IOUtils.quietClose(rs, stmt, conn);
- }
- }
-
- @Override
- public int getNumItems() throws TasteException {
- return getNumThings("items", getNumItemsSQL);
- }
-
- @Override
- public int getNumUsers() throws TasteException {
- return getNumThings("users", getNumUsersSQL);
- }
-
- @Override
- public int getNumUsersWithPreferenceFor(Object... itemIDs) throws TasteException {
- if (itemIDs == null) {
- throw new IllegalArgumentException("itemIDs is null");
- }
- int length = itemIDs.length;
- if (length == 0 || length > 2) {
- throw new IllegalArgumentException("Illegal number of item IDs: " + length);
- }
- return length == 1 ?
- getNumThings("user preferring item", getNumPreferenceForItemSQL, itemIDs) :
- getNumThings("user preferring items", getNumPreferenceForItemsSQL, itemIDs);
- }
-
-
- private int getNumThings(String name, String sql, Object... args) throws TasteException {
- log.debug("Retrieving number of {} in model", name);
- Connection conn = null;
- PreparedStatement stmt = null;
- ResultSet rs = null;
- try {
- conn = dataSource.getConnection();
- stmt = conn.prepareStatement(sql);
- if (args != null) {
- for (int i = 1; i <= args.length; i++) {
- stmt.setObject(i, args[i - 1]);
- }
- }
- log.debug("Executing SQL query: {}", sql);
- rs = stmt.executeQuery();
- rs.next();
- return rs.getInt(1);
- } catch (SQLException sqle) {
- log.warn("Exception while retrieving number of " + name, sqle);
- throw new TasteException(sqle);
- } finally {
- IOUtils.quietClose(rs, stmt, conn);
- }
+ public Iterable<? extends User> getUsers() throws TasteException {
+ log.debug("Retrieving all users...");
+ return new IteratorIterable<User>(new ResultSetUserIterator(getDataSource(), getUsersSQL));
}
@Override
@@ -431,25 +138,23 @@
if (userID == null || itemID == null) {
throw new IllegalArgumentException("userID or itemID is null");
}
- if (Double.isNaN(value)) {
+ if (!Double.isNaN(value)) {
throw new IllegalArgumentException("Invalid value: " + value);
}
if (log.isDebugEnabled()) {
- log.debug("Setting preference for user '" + userID + "', item '" + itemID + "', value " + value);
+ log.debug("Setting preference for user '" + userID + "', item '" + itemID);
}
Connection conn = null;
PreparedStatement stmt = null;
try {
- conn = dataSource.getConnection();
+ conn = getDataSource().getConnection();
stmt = conn.prepareStatement(setPreferenceSQL);
stmt.setObject(1, userID);
stmt.setObject(2, itemID);
- stmt.setDouble(3, value);
- stmt.setDouble(4, value);
log.debug("Executing SQL update: {}", setPreferenceSQL);
stmt.executeUpdate();
@@ -463,90 +168,42 @@
}
@Override
- public void removePreference(Object userID, Object itemID)
- throws TasteException {
- if (userID == null || itemID == null) {
- throw new IllegalArgumentException("userID or itemID is null");
- }
-
- log.debug("Removing preference for user '{}', item '{}'", userID, itemID);
-
+ protected List<? extends Preference> doGetPreferencesForItem(Object itemID) throws TasteException {
+ log.debug("Retrieving preferences for item ID '{}'", itemID);
+ Item item = getItem(itemID, true);
Connection conn = null;
PreparedStatement stmt = null;
-
+ ResultSet rs = null;
try {
- conn = dataSource.getConnection();
-
- stmt = conn.prepareStatement(removePreferenceSQL);
- stmt.setObject(1, userID);
- stmt.setObject(2, itemID);
-
- log.debug("Executing SQL update: {}", removePreferenceSQL);
- stmt.executeUpdate();
+ conn = getDataSource().getConnection();
+ stmt = conn.prepareStatement(getPrefsForItemSQL);
+ stmt.setObject(1, itemID);
+ log.debug("Executing SQL query: {}", getPrefsForItemSQL);
+ rs = stmt.executeQuery();
+ List<Preference> prefs = new ArrayList<Preference>();
+ while (rs.next()) {
+ String userID = rs.getString(2);
+ Preference pref = buildPreference(buildUser(userID, (FastSet<Object>) null), item);
+ prefs.add(pref);
+ }
+ return prefs;
} catch (SQLException sqle) {
- log.warn("Exception while removing preference", sqle);
+ log.warn("Exception while retrieving prefs for item", sqle);
throw new TasteException(sqle);
} finally {
- IOUtils.quietClose(null, stmt, conn);
+ IOUtils.quietClose(rs, stmt, conn);
}
}
- @Override
- public void refresh(Collection<Refreshable> alreadyRefreshed) {
- // do nothing
- }
-
-
- private void addPreference(ResultSet rs, Collection<Preference> prefs)
- throws SQLException {
- Item item = buildItem(rs.getString(1));
- double preferenceValue = rs.getDouble(2);
- prefs.add(buildPreference(null, item, preferenceValue));
- }
-
- /**
- * <p>Default implementation which returns a new {@link GenericUser} with {@link String} IDs.
- * Subclasses may override to return a different {@link User} implementation.</p>
- *
- * @param id user ID
- * @param prefs user preferences
- * @return {@link GenericUser} by default
- */
- protected User buildUser(String id, List<Preference> prefs) {
- return new GenericUser<String>(id, prefs);
- }
-
- /**
- * <p>Default implementation which returns a new {@link GenericItem} with {@link String} IDs.
- * Subclasses may override to return a different {@link Item} implementation.</p>
- *
- * @param id item ID
- * @return {@link GenericItem} by default
- */
- protected Item buildItem(String id) {
- return new GenericItem<String>(id);
+ protected User buildUser(String id, FastSet<Object> itemIDs) {
+ return new BooleanPrefUser<String>(id, itemIDs);
}
- /**
- * Subclasses may override to return a different {@link Preference} implementation.
- *
- * @param user {@link User}
- * @param item {@link Item}
- * @return {@link GenericPreference} by default
- */
- protected Preference buildPreference(User user, Item item, double value) {
- return new GenericPreference(user, item, value);
+ protected Preference buildPreference(User user, Item item) {
+ return new BooleanPreference(user, item);
}
- /**
- * <p>An {@link java.util.Iterator} which returns {@link org.apache.mahout.cf.taste.model.User}s from a
- * {@link java.sql.ResultSet}. This is a useful
- * way to iterate over all user data since it does not require all data to be read into memory
- * at once. It does however require that the DB connection be held open. Note that this class will
- * only release database resources after {@link #hasNext()} has been called and has returned false;
- * callers should make sure to "drain" the entire set of data to avoid tying up database resources.</p>
- */
private final class ResultSetUserIterator implements Iterator<User> {
private final Connection connection;
@@ -556,7 +213,7 @@
private ResultSetUserIterator(DataSource dataSource, String getUsersSQL) throws TasteException {
try {
- connection = dataSource.getConnection();
+ connection = getDataSource().getConnection();
// These settings should enable the ResultSet to be iterated in both directions
statement = connection.prepareStatement(getUsersSQL,
ResultSet.TYPE_SCROLL_INSENSITIVE,
@@ -598,11 +255,11 @@
}
String currentUserID = null;
- List<Preference> prefs = new ArrayList<Preference>();
+ FastSet<Object> itemIDs = new FastSet<Object>();
try {
while (resultSet.next()) {
- String userID = resultSet.getString(3);
+ String userID = resultSet.getString(2);
if (currentUserID == null) {
currentUserID = userID;
}
@@ -614,7 +271,7 @@
break;
}
// else add a new preference for the current user
- addPreference(resultSet, prefs);
+ itemIDs.add(resultSet.getString(1));
}
} catch (SQLException sqle) {
// No good way to handle this since we can't throw an exception
@@ -628,92 +285,7 @@
throw new NoSuchElementException();
}
- return buildUser(currentUserID, prefs);
- }
-
- /**
- * @throws UnsupportedOperationException
- */
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
- }
-
- private void close() {
- closed = true;
- IOUtils.quietClose(resultSet, statement, connection);
- }
-
- }
-
- /**
- * <p>An {@link java.util.Iterator} which returns {@link org.apache.mahout.cf.taste.model.Item}s from a
- * {@link java.sql.ResultSet}. This is a useful way to iterate over all user data since it does not require
- * all data to be read into memory at once. It does however require that the DB connection be held open. Note
- * that this class will only release database resources after {@link #hasNext()} has been called and has returned
- * <code>false</code>; callers should make sure to "drain" the entire set of data to avoid tying up database
- * resources.</p>
- */
- private final class ResultSetItemIterator implements Iterator<Item> {
-
- private final Connection connection;
- private final PreparedStatement statement;
- private final ResultSet resultSet;
- private boolean closed;
-
- private ResultSetItemIterator(DataSource dataSource, String getItemsSQL) throws TasteException {
- try {
- connection = dataSource.getConnection();
- statement = connection.prepareStatement(getItemsSQL);
- statement.setFetchDirection(ResultSet.FETCH_FORWARD);
- log.debug("Executing SQL query: {}", getItemsSQL);
- resultSet = statement.executeQuery();
- } catch (SQLException sqle) {
- close();
- throw new TasteException(sqle);
- }
- }
-
- @Override
- public boolean hasNext() {
- boolean nextExists = false;
- if (!closed) {
- try {
- // No more results if cursor is pointing at last row, or after
- // Thanks to Rolf W. for pointing out an earlier bug in this condition
- if (resultSet.isLast() || resultSet.isAfterLast()) {
- close();
- } else {
- nextExists = true;
- }
- } catch (SQLException sqle) {
- log.warn("Unexpected exception while accessing ResultSet; continuing...", sqle);
- close();
- }
- }
- return nextExists;
- }
-
- @Override
- public Item next() {
-
- if (closed) {
- throw new NoSuchElementException();
- }
-
- try {
- if (resultSet.next()) {
- return buildItem(resultSet.getString(1));
- } else {
- throw new NoSuchElementException();
- }
- } catch (SQLException sqle) {
- // No good way to handle this since we can't throw an exception
- log.warn("Exception while iterating over items", sqle);
- close();
- throw new NoSuchElementException("Can't retrieve more due to exception: " + sqle);
- }
-
+ return buildUser(currentUserID, itemIDs);
}
/**
@@ -731,4 +303,4 @@
}
-}
+}
\ No newline at end of file
Modified: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java?rev=791905&r1=791904&r2=791905&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/AbstractJDBCDataModel.java Tue Jul 7 16:45:25 2009
@@ -346,7 +346,7 @@
return list.toArray(new Preference[list.size()]);
}
- private List<? extends Preference> doGetPreferencesForItem(Object itemID) throws TasteException {
+ protected List<? extends Preference> doGetPreferencesForItem(Object itemID) throws TasteException {
log.debug("Retrieving preferences for item ID '{}'", itemID);
Item item = getItem(itemID, true);
Connection conn = null;
Copied: lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java (from r791748, lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java)
URL: http://svn.apache.org/viewvc/lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java?p2=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java&p1=lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java&r1=791748&r2=791905&rev=791905&view=diff
==============================================================================
--- lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLJDBCDataModel.java (original)
+++ lucene/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/MySQLBooleanPrefJDBCDataModel.java Tue Jul 7 16:45:25 2009
@@ -18,84 +18,42 @@
package org.apache.mahout.cf.taste.impl.model.jdbc;
import org.apache.mahout.cf.taste.common.TasteException;
-import org.apache.mahout.cf.taste.model.DataModel;
import javax.sql.DataSource;
/**
- * <p>A {@link DataModel} backed by a MySQL database and accessed via JDBC. It may work with other
- * JDBC databases. By default, this class assumes that there is a {@link DataSource} available under the
- * JNDI name "jdbc/taste", which gives access to a database with a "taste_preferences" table with the
- * following schema:</p>
- *
- * <table>
- * <tr><th>user_id</th><th>item_id</th><th>preference</th></tr>
- * <tr><td>ABC</td><td>123</td><td>0.9</td></tr>
- * <tr><td>ABC</td><td>456</td><td>0.1</td></tr>
- * <tr><td>DEF</td><td>123</td><td>0.2</td></tr>
- * <tr><td>DEF</td><td>789</td><td>0.3</td></tr>
- * </table>
- *
- * <p><code>user_id</code> must have a type compatible with the Java <code>String</code> type.
- * <code>item_id</code> must have a type compatible with the Java <code>String</code> type.
- * <code>preference</code> must have a type compatible with the Java <code>double</code> type.
- * For example, the following command sets up a suitable table in MySQL, complete with
- * primary key and indexes:</p>
+ * <p>See also {@link MySQLJDBCDataModel} -- same except deals with a table without preference info:</p>
*
* <pre>
* CREATE TABLE taste_preferences (
* user_id VARCHAR(10) NOT NULL,
* item_id VARCHAR(10) NOT NULL,
- * preference FLOAT NOT NULL,
* PRIMARY KEY (user_id, item_id),
* INDEX (user_id),
* INDEX (item_id)
* )
* </pre>
- *
- * <h3>Performance Notes</h3>
- *
- * <p>See the notes in {@link AbstractJDBCDataModel} regarding using connection pooling. It's pretty vital
- * to performance.</p>
- *
- * <p>Some experimentation suggests that MySQL's InnoDB engine is faster than MyISAM for these kinds of
- * applications. While MyISAM is the default and, I believe, generally considered the lighter-weight and faster
- * of the two engines, my guess is the row-level locking of InnoDB helps here. Your mileage may vary.</p>
- *
- * <p>Here are some key settings that can be tuned for MySQL, and suggested size for a data set of around
- * 1 million elements:</p>
- *
- * <ul>
- * <li>innodb_buffer_pool_size=64M</li>
- * <li>myisam_sort_buffer_size=64M</li>
- * <li>query_cache_limit=64M</li>
- * <li>query_cache_min_res_unit=512K</li>
- * <li>query_cache_type=1</li>
- * <li>query_cache_size=64M</li>
- * </ul>
- *
- * <p>Thanks to Amila Jayasooriya for contributing MySQL notes above as part of Google Summer of Code 2007.</p>
*/
-public class MySQLJDBCDataModel extends AbstractJDBCDataModel {
+public class MySQLBooleanPrefJDBCDataModel extends AbstractBooleanPrefJDBCDataModel {
/**
- * <p>Creates a {@link MySQLJDBCDataModel} using the default {@link DataSource}
+ * <p>Creates a {@link MySQLBooleanPrefJDBCDataModel} using the default {@link javax.sql.DataSource}
* (named {@link #DEFAULT_DATASOURCE_NAME} and default table/column names.</p>
*
- * @throws TasteException if {@link DataSource} can't be found
+ * @throws org.apache.mahout.cf.taste.common.TasteException if {@link javax.sql.DataSource} can't be found
*/
- public MySQLJDBCDataModel() throws TasteException {
+ public MySQLBooleanPrefJDBCDataModel() throws TasteException {
this(DEFAULT_DATASOURCE_NAME);
}
/**
- * <p>Creates a {@link MySQLJDBCDataModel} using the default {@link DataSource}
+ * <p>Creates a {@link MySQLBooleanPrefJDBCDataModel} using the default {@link javax.sql.DataSource}
* found under the given name, and using default table/column names.</p>
*
- * @param dataSourceName name of {@link DataSource} to look up
- * @throws TasteException if {@link DataSource} can't be found
+ * @param dataSourceName name of {@link javax.sql.DataSource} to look up
+ * @throws org.apache.mahout.cf.taste.common.TasteException if {@link javax.sql.DataSource} can't be found
*/
- public MySQLJDBCDataModel(String dataSourceName) throws TasteException {
+ public MySQLBooleanPrefJDBCDataModel(String dataSourceName) throws TasteException {
this(lookupDataSource(dataSourceName),
DEFAULT_PREFERENCE_TABLE,
DEFAULT_USER_ID_COLUMN,
@@ -104,12 +62,12 @@
}
/**
- * <p>Creates a {@link MySQLJDBCDataModel} using the given {@link DataSource}
+ * <p>Creates a {@link MySQLBooleanPrefJDBCDataModel} using the given {@link javax.sql.DataSource}
* and default table/column names.</p>
*
- * @param dataSource {@link DataSource} to use
+ * @param dataSource {@link javax.sql.DataSource} to use
*/
- public MySQLJDBCDataModel(DataSource dataSource) {
+ public MySQLBooleanPrefJDBCDataModel(DataSource dataSource) {
this(dataSource,
DEFAULT_PREFERENCE_TABLE,
DEFAULT_USER_ID_COLUMN,
@@ -118,46 +76,43 @@
}
/**
- * <p>Creates a {@link MySQLJDBCDataModel} using the given {@link DataSource}
+ * <p>Creates a {@link MySQLBooleanPrefJDBCDataModel} using the given {@link javax.sql.DataSource}
* and default table/column names.</p>
*
- * @param dataSource {@link DataSource} to use
+ * @param dataSource {@link javax.sql.DataSource} to use
* @param preferenceTable name of table containing preference data
* @param userIDColumn user ID column name
* @param itemIDColumn item ID column name
* @param preferenceColumn preference column name
*/
- public MySQLJDBCDataModel(DataSource dataSource,
- String preferenceTable,
- String userIDColumn,
- String itemIDColumn,
- String preferenceColumn) {
+ public MySQLBooleanPrefJDBCDataModel(DataSource dataSource,
+ String preferenceTable,
+ String userIDColumn,
+ String itemIDColumn,
+ String preferenceColumn) {
super(dataSource,
preferenceTable,
userIDColumn,
itemIDColumn,
preferenceColumn,
// getUserSQL
- "SELECT " + itemIDColumn + ", " + preferenceColumn + " FROM " + preferenceTable +
- " WHERE " + userIDColumn + "=? ORDER BY " + itemIDColumn,
+ "SELECT " + itemIDColumn + " FROM " + preferenceTable + " WHERE " + userIDColumn + "=?",
// getNumItemsSQL
"SELECT COUNT(DISTINCT " + itemIDColumn + ") FROM " + preferenceTable,
// getNumUsersSQL
"SELECT COUNT(DISTINCT " + userIDColumn + ") FROM " + preferenceTable,
// setPreferenceSQL
- "INSERT INTO " + preferenceTable + '(' + userIDColumn + ',' + itemIDColumn + ',' + preferenceColumn +
- ") VALUES (?,?,?) ON DUPLICATE KEY UPDATE " + preferenceColumn + "=?",
+ "INSERT IGNORE INTO " + preferenceTable + '(' + userIDColumn + ',' + itemIDColumn + ") VALUES (?,?)",
// removePreference SQL
"DELETE FROM " + preferenceTable + " WHERE " + userIDColumn + "=? AND " + itemIDColumn + "=?",
// getUsersSQL
- "SELECT " + itemIDColumn + ", " + preferenceColumn + ", " + userIDColumn + " FROM " +
- preferenceTable + " ORDER BY " + userIDColumn + ", " + itemIDColumn,
+ "SELECT " + itemIDColumn + ", " + userIDColumn + " FROM " + preferenceTable + " ORDER BY " + userIDColumn,
// getItemsSQL
"SELECT DISTINCT " + itemIDColumn + " FROM " + preferenceTable + " ORDER BY " + itemIDColumn,
// getItemSQL
"SELECT 1 FROM " + preferenceTable + " WHERE " + itemIDColumn + "=?",
// getPrefsForItemSQL
- "SELECT " + preferenceColumn + ", " + userIDColumn + " FROM " +
+ "SELECT " + userIDColumn + " FROM " +
preferenceTable + " WHERE " + itemIDColumn + "=? ORDER BY " + userIDColumn,
// getNumPreferenceForItemSQL
"SELECT COUNT(1) FROM " + preferenceTable + " WHERE " + itemIDColumn + "=?",
@@ -167,4 +122,4 @@
"WHERE tp1." + itemIDColumn + "=? and tp2." + itemIDColumn + "=?");
}
-}
+}
\ No newline at end of file