You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mahout.apache.org by sr...@apache.org on 2010/11/28 20:57:01 UTC
svn commit: r1039936 - in /mahout/trunk/core/src:
main/java/org/apache/mahout/cf/taste/impl/common/
main/java/org/apache/mahout/cf/taste/impl/model/jdbc/
main/java/org/apache/mahout/cf/taste/impl/recommender/
main/java/org/apache/mahout/cf/taste/impl/s...
Author: srowen
Date: Sun Nov 28 19:57:00 2010
New Revision: 1039936
URL: http://svn.apache.org/viewvc?rev=1039936&view=rev
Log:
Refine some of the refresh logic and add a handy way to partly clear Caches
Added:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/LongPairMatchPredicate.java
Modified:
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/Cache.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/ReloadFromJDBCDataModel.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericItemBasedRecommender.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericUserBasedRecommender.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingItemSimilarity.java
mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingUserSimilarity.java
mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/common/FastMapTest.java
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/Cache.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/Cache.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/Cache.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/common/Cache.java Sun Nov 28 19:57:00 2010
@@ -20,6 +20,8 @@ package org.apache.mahout.cf.taste.impl.
import com.google.common.base.Preconditions;
import org.apache.mahout.cf.taste.common.TasteException;
+import java.util.Iterator;
+
/**
* <p>
* An efficient Map-like class which caches values for keys. Values are not "put" into a ;
@@ -108,6 +110,36 @@ public final class Cache<K,V> implements
cache.remove(key);
}
}
+
+ /**
+ * Clears all cache entries whose key matches the given predicate.
+ */
+ public void removeKeysMatching(MatchPredicate<K> predicate) {
+ synchronized (cache) {
+ Iterator<K> it = cache.keySet().iterator();
+ while (it.hasNext()) {
+ K key = it.next();
+ if (predicate.matches(key)) {
+ it.remove();
+ }
+ }
+ }
+ }
+
+ /**
+ * Clears all cache entries whose value matches the given predicate.
+ */
+ public void removeValueMatching(MatchPredicate<V> predicate) {
+ synchronized (cache) {
+ Iterator<V> it = cache.values().iterator();
+ while (it.hasNext()) {
+ V value = it.next();
+ if (predicate.matches(value)) {
+ it.remove();
+ }
+ }
+ }
+ }
/**
* <p>
@@ -135,5 +167,12 @@ public final class Cache<K,V> implements
public String toString() {
return "Cache[retriever:" + retriever + ']';
}
+
+ /**
+ * Used by {#link #removeKeysMatching(Object)} to decide things that are matching.
+ */
+ public interface MatchPredicate<T> {
+ boolean matches(T thing);
+ }
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/ReloadFromJDBCDataModel.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/ReloadFromJDBCDataModel.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/ReloadFromJDBCDataModel.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/model/jdbc/ReloadFromJDBCDataModel.java Sun Nov 28 19:57:00 2010
@@ -22,6 +22,7 @@ import org.apache.mahout.cf.taste.common
import org.apache.mahout.cf.taste.common.TasteException;
import org.apache.mahout.cf.taste.impl.common.FastIDSet;
import org.apache.mahout.cf.taste.impl.common.LongPrimitiveIterator;
+import org.apache.mahout.cf.taste.impl.common.RefreshHelper;
import org.apache.mahout.cf.taste.impl.model.GenericBooleanPrefDataModel;
import org.apache.mahout.cf.taste.impl.model.GenericDataModel;
import org.apache.mahout.cf.taste.model.DataModel;
@@ -31,6 +32,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Collection;
+import java.util.concurrent.Callable;
/**
* A {@link DataModel} which loads, and can re-load, data from a JDBC-backed {@link JDBCDataModel} into memory, as a
@@ -43,11 +45,20 @@ public final class ReloadFromJDBCDataMod
private DataModel delegateInMemory;
private final JDBCDataModel delegate;
+ private final RefreshHelper refreshHelper;
public ReloadFromJDBCDataModel(JDBCDataModel delegate) throws TasteException {
Preconditions.checkNotNull(delegate);
this.delegate = delegate;
- refresh(null);
+ refreshHelper = new RefreshHelper(new Callable<Void>() {
+ @Override
+ public Void call() {
+ reload();
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+ });
+ refreshHelper.addDependency(delegate);
+ reload();
if (delegateInMemory == null) {
throw new TasteException("Failed to load data into memory");
}
@@ -55,7 +66,10 @@ public final class ReloadFromJDBCDataMod
@Override
public void refresh(Collection<Refreshable> alreadyRefreshed) {
- delegate.refresh(alreadyRefreshed);
+ refreshHelper.refresh(alreadyRefreshed);
+ }
+
+ private void reload() {
try {
// Load new in-memory representation,
log.info("Loading new JDBC delegate data...");
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericItemBasedRecommender.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericItemBasedRecommender.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericItemBasedRecommender.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericItemBasedRecommender.java Sun Nov 28 19:57:00 2010
@@ -21,6 +21,7 @@ import org.apache.mahout.cf.taste.recomm
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.Callable;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
@@ -78,7 +79,13 @@ public class GenericItemBasedRecommender
super(dataModel, candidateItemsStrategy);
Preconditions.checkArgument(similarity != null, "similarity is null");
this.similarity = similarity;
- this.refreshHelper = new RefreshHelper(null);
+ this.refreshHelper = new RefreshHelper(new Callable<Void>() {
+ @Override
+ public Void call() {
+ capper = buildCapper();
+ return null;
+ }
+ });
refreshHelper.addDependency(dataModel);
refreshHelper.addDependency(similarity);
capper = buildCapper();
@@ -219,7 +226,6 @@ public class GenericItemBasedRecommender
@Override
public void refresh(Collection<Refreshable> alreadyRefreshed) {
refreshHelper.refresh(alreadyRefreshed);
- capper = buildCapper();
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericUserBasedRecommender.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericUserBasedRecommender.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericUserBasedRecommender.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/recommender/GenericUserBasedRecommender.java Sun Nov 28 19:57:00 2010
@@ -20,6 +20,7 @@ package org.apache.mahout.cf.taste.impl.
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.Callable;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
@@ -60,7 +61,13 @@ public class GenericUserBasedRecommender
Preconditions.checkArgument(neighborhood != null, "neighborhood is null");
this.neighborhood = neighborhood;
this.similarity = similarity;
- this.refreshHelper = new RefreshHelper(null);
+ this.refreshHelper = new RefreshHelper(new Callable<Void>() {
+ @Override
+ public Void call() {
+ capper = buildCapper();
+ return null;
+ }
+ });
refreshHelper.addDependency(dataModel);
refreshHelper.addDependency(similarity);
refreshHelper.addDependency(neighborhood);
@@ -171,7 +178,6 @@ public class GenericUserBasedRecommender
@Override
public void refresh(Collection<Refreshable> alreadyRefreshed) {
refreshHelper.refresh(alreadyRefreshed);
- capper = buildCapper();
}
@Override
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingItemSimilarity.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingItemSimilarity.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingItemSimilarity.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingItemSimilarity.java Sun Nov 28 19:57:00 2010
@@ -18,6 +18,7 @@
package org.apache.mahout.cf.taste.impl.similarity;
import java.util.Collection;
+import java.util.concurrent.Callable;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
@@ -29,11 +30,13 @@ import org.apache.mahout.cf.taste.simila
import org.apache.mahout.common.LongPair;
import com.google.common.base.Preconditions;
-/** Caches the results from an underlying {@link ItemSimilarity} implementation. */
+/**
+ * Caches the results from an underlying {@link ItemSimilarity} implementation.
+ */
public final class CachingItemSimilarity implements ItemSimilarity {
- private final ItemSimilarity similarity;
private final Cache<LongPair,Double> similarityCache;
+ private final RefreshHelper refreshHelper;
/**
* Creates a {@link CachingItemSimilarity} on top of the given {@link ItemSimilarity}.
@@ -49,8 +52,15 @@ public final class CachingItemSimilarity
*/
public CachingItemSimilarity(ItemSimilarity similarity, int maxCacheSize) {
Preconditions.checkArgument(similarity != null, "similarity is null");
- this.similarity = similarity;
this.similarityCache = new Cache<LongPair,Double>(new SimilarityRetriever(similarity), maxCacheSize);
+ this.refreshHelper = new RefreshHelper(new Callable<Void>() {
+ @Override
+ public Void call() {
+ similarityCache.clear();
+ return null;
+ }
+ });
+ refreshHelper.addDependency(similarity);
}
@Override
@@ -71,9 +81,11 @@ public final class CachingItemSimilarity
@Override
public void refresh(Collection<Refreshable> alreadyRefreshed) {
- similarityCache.clear();
- alreadyRefreshed = RefreshHelper.buildRefreshed(alreadyRefreshed);
- RefreshHelper.maybeRefresh(alreadyRefreshed, similarity);
+ refreshHelper.refresh(alreadyRefreshed);
+ }
+
+ public void clearCacheForItem(long itemID) {
+ similarityCache.removeKeysMatching(new LongPairMatchPredicate(itemID));
}
private static final class SimilarityRetriever implements Retriever<LongPair,Double> {
@@ -88,5 +100,5 @@ public final class CachingItemSimilarity
return similarity.itemSimilarity(key.getFirst(), key.getSecond());
}
}
-
+
}
Modified: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingUserSimilarity.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingUserSimilarity.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingUserSimilarity.java (original)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/CachingUserSimilarity.java Sun Nov 28 19:57:00 2010
@@ -18,6 +18,7 @@
package org.apache.mahout.cf.taste.impl.similarity;
import java.util.Collection;
+import java.util.concurrent.Callable;
import org.apache.mahout.cf.taste.common.Refreshable;
import org.apache.mahout.cf.taste.common.TasteException;
@@ -32,13 +33,13 @@ import org.apache.mahout.common.LongPair
import com.google.common.base.Preconditions;
/**
- * Caches the results from an underlying {@link org.apache.mahout.cf.taste.similarity.UserSimilarity}
- * implementation.
+ * Caches the results from an underlying {@link UserSimilarity} implementation.
*/
public final class CachingUserSimilarity implements UserSimilarity {
private final UserSimilarity similarity;
private final Cache<LongPair,Double> similarityCache;
+ private final RefreshHelper refreshHelper;
/**
* Creates a {@link CachingUserSimilarity} on top of the given {@link UserSimilarity}.
@@ -56,6 +57,14 @@ public final class CachingUserSimilarity
Preconditions.checkArgument(similarity != null, "similarity is null");
this.similarity = similarity;
this.similarityCache = new Cache<LongPair,Double>(new SimilarityRetriever(similarity), maxCacheSize);
+ this.refreshHelper = new RefreshHelper(new Callable<Void>() {
+ @Override
+ public Void call() {
+ similarityCache.clear();
+ return null;
+ }
+ });
+ refreshHelper.addDependency(similarity);
}
@Override
@@ -69,12 +78,14 @@ public final class CachingUserSimilarity
similarityCache.clear();
similarity.setPreferenceInferrer(inferrer);
}
+
+ public void clearCacheForUser(long userID) {
+ similarityCache.removeKeysMatching(new LongPairMatchPredicate(userID));
+ }
@Override
public void refresh(Collection<Refreshable> alreadyRefreshed) {
- similarityCache.clear();
- alreadyRefreshed = RefreshHelper.buildRefreshed(alreadyRefreshed);
- RefreshHelper.maybeRefresh(alreadyRefreshed, similarity);
+ refreshHelper.refresh(alreadyRefreshed);
}
private static final class SimilarityRetriever implements Retriever<LongPair,Double> {
Added: mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/LongPairMatchPredicate.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/LongPairMatchPredicate.java?rev=1039936&view=auto
==============================================================================
--- mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/LongPairMatchPredicate.java (added)
+++ mahout/trunk/core/src/main/java/org/apache/mahout/cf/taste/impl/similarity/LongPairMatchPredicate.java Sun Nov 28 19:57:00 2010
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mahout.cf.taste.impl.similarity;
+
+import org.apache.mahout.cf.taste.impl.common.Cache;
+import org.apache.mahout.common.LongPair;
+
+/**
+ * A {@link Cache.MatchPredicate} which will match an ID against either element of a
+ * {@link LongPair}.
+ */
+final class LongPairMatchPredicate implements Cache.MatchPredicate<LongPair> {
+
+ private final long id;
+
+ LongPairMatchPredicate(long id) {
+ this.id = id;
+ }
+
+ @Override
+ public boolean matches(LongPair pair) {
+ return pair.getFirst() == id || pair.getSecond() == id;
+ }
+
+}
Modified: mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/common/FastMapTest.java
URL: http://svn.apache.org/viewvc/mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/common/FastMapTest.java?rev=1039936&r1=1039935&r2=1039936&view=diff
==============================================================================
--- mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/common/FastMapTest.java (original)
+++ mahout/trunk/core/src/test/java/org/apache/mahout/cf/taste/impl/common/FastMapTest.java Sun Nov 28 19:57:00 2010
@@ -24,6 +24,7 @@ import org.junit.Test;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
@@ -131,6 +132,16 @@ public final class FastMapTest extends T
Set<String> actual = map.keySet();
assertTrue(expected.containsAll(actual));
assertTrue(actual.containsAll(expected));
+ Iterator<String> it = actual.iterator();
+ while (it.hasNext()) {
+ String value = it.next();
+ if (!value.equals("baz")) {
+ it.remove();
+ }
+ }
+ assertTrue(map.containsKey("baz"));
+ assertFalse(map.containsKey("foo"));
+ assertFalse(map.containsKey("alpha"));
}
@Test
@@ -143,6 +154,16 @@ public final class FastMapTest extends T
Collection<String> actual = map.values();
assertTrue(expected.containsAll(actual));
assertTrue(actual.containsAll(expected));
+ Iterator<String> it = actual.iterator();
+ while (it.hasNext()) {
+ String value = it.next();
+ if (!value.equals("bang")) {
+ it.remove();
+ }
+ }
+ assertTrue(map.containsValue("bang"));
+ assertFalse(map.containsValue("bar"));
+ assertFalse(map.containsValue("beta"));
}
@Test