You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/11/20 20:35:10 UTC
git commit: try with ehcache to get better memory behaviour
Updated Branches:
refs/heads/develop 1fc4aafcb -> 578655489
try with ehcache to get better memory behaviour
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/57865548
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/57865548
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/57865548
Branch: refs/heads/develop
Commit: 578655489fcf55fc8b200d23be22ceca7817a127
Parents: 1fc4aaf
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Wed Nov 20 20:35:06 2013 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Wed Nov 20 20:35:06 2013 +0100
----------------------------------------------------------------------
.../kiwi/loader/generic/CacheWeigher.java | 60 ----------
.../kiwi/loader/generic/KiWiHandler.java | 112 ++++++++++++-------
.../kiwi/loader/generic/Statistics.java | 10 +-
3 files changed, 78 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/57865548/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/CacheWeigher.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/CacheWeigher.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/CacheWeigher.java
deleted file mode 100644
index a112806..0000000
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/CacheWeigher.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.marmotta.kiwi.loader.generic;
-
-import com.google.common.cache.Weigher;
-import com.google.common.io.ByteStreams;
-import com.google.common.io.CountingOutputStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.IOException;
-import java.io.ObjectOutputStream;
-import java.io.Serializable;
-
-/**
- * Add file description here!
- *
- * @author Sebastian Schaffert (sschaffert@apache.org)
- */
-public class CacheWeigher<K extends Serializable,V extends Serializable> implements Weigher<K,V> {
-
- private static Logger log = LoggerFactory.getLogger(CacheWeigher.class);
-
- /**
- * Returns the weight of a cache entry. There is no unit for entry weights; rather they are simply
- * relative to each other.
- *
- * @return the weight of the entry; must be non-negative
- */
- @Override
- public int weigh(K key, V value) {
- try {
- CountingOutputStream counter = new CountingOutputStream(ByteStreams.nullOutputStream());
- ObjectOutputStream out = new ObjectOutputStream(counter);
- out.writeObject(key);
- out.writeObject(value);
- int result = (int) counter.getCount();
- out.close();
- return result + 16 + 48; // add 16 + 48 bytes for the cache maintenance information itself
- } catch (IOException e) {
- log.error("could not determine object size ...");
- return 1024; // assume 1 kB
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/57865548/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
index a6fa6fb..135d500 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
@@ -3,6 +3,14 @@ package org.apache.marmotta.kiwi.loader.generic;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import net.sf.ehcache.Cache;
+import net.sf.ehcache.CacheManager;
+import net.sf.ehcache.Element;
+import net.sf.ehcache.config.CacheConfiguration;
+import net.sf.ehcache.config.MemoryUnit;
+import net.sf.ehcache.constructs.blocking.CacheEntryFactory;
+import net.sf.ehcache.constructs.blocking.SelfPopulatingCache;
+import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.apache.marmotta.commons.sesame.model.Namespaces;
import org.apache.marmotta.commons.util.DateUtils;
import org.apache.marmotta.kiwi.loader.KiWiLoaderConfiguration;
@@ -57,14 +65,15 @@ public class KiWiHandler implements RDFHandler {
protected KiWiLoaderConfiguration config;
- protected LoadingCache<Literal, KiWiLiteral> literalCache;
- protected LoadingCache<URI, KiWiUriResource> uriCache;
- protected LoadingCache<BNode, KiWiAnonResource> bnodeCache;
+ protected SelfPopulatingCache literalCache;
+ protected SelfPopulatingCache uriCache;
+ protected SelfPopulatingCache bnodeCache;
protected LoadingCache<String,Locale> localeCache;
// if non-null, all imported statements will have this context (regardless whether they specified a different context)
private KiWiResource overrideContext;
+ private CacheManager cacheManager;
private Statistics statistics;
@@ -72,6 +81,8 @@ public class KiWiHandler implements RDFHandler {
this.config = config;
this.store = store;
+ this.cacheManager = CacheManager.create();
+
long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
long freeMemory = Runtime.getRuntime().maxMemory() - usedMemory;
long litCacheSize = (long) (freeMemory * LITERAL_CACHE_MEMORY_PERCENTAGE);
@@ -82,39 +93,53 @@ public class KiWiHandler implements RDFHandler {
log.info("calculated cache sizes: uri={}B, literal={}B, bnode={}B", formatSize(uriCacheSize), formatSize(litCacheSize), formatSize(bnodeCacheSize));
+ Cache literalBaseCache = new Cache(
+ new CacheConfiguration("literalCache",0)
+ .maxBytesLocalHeap(litCacheSize, MemoryUnit.BYTES)
+ .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU)
+ .eternal(false)
+ .statistics(true)
+ );
+ cacheManager.addCache(literalBaseCache);
+
+ Cache uriBaseCache = new Cache(
+ new CacheConfiguration("uriCache",0)
+ .maxBytesLocalHeap(uriCacheSize, MemoryUnit.BYTES)
+ .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU)
+ .eternal(false)
+ .statistics(true)
+ );
+ cacheManager.addCache(uriBaseCache);
+
+ Cache bnodeBaseCache = new Cache(
+ new CacheConfiguration("bnodeCache",0)
+ .maxBytesLocalHeap(bnodeCacheSize, MemoryUnit.BYTES)
+ .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LFU)
+ .eternal(false)
+ .statistics(true)
+ );
+ cacheManager.addCache(bnodeBaseCache);
+
+ this.literalCache = new SelfPopulatingCache(cacheManager.getCache("literalCache"), new CacheEntryFactory() {
+ @Override
+ public Object createEntry(Object key) throws Exception {
+ return createLiteral((Literal)key);
+ }
+ });
- this.literalCache = CacheBuilder.newBuilder()
- .recordStats()
- .maximumWeight(litCacheSize)
- .weigher(new CacheWeigher<Literal, KiWiLiteral>())
- .build(new CacheLoader<Literal, KiWiLiteral>() {
- @Override
- public KiWiLiteral load(Literal l) throws Exception {
- return createLiteral(l);
- }
- });
-
- this.uriCache = CacheBuilder.newBuilder()
- .recordStats()
- .maximumWeight(uriCacheSize)
- .weigher(new CacheWeigher<URI, KiWiUriResource>())
- .build(new CacheLoader<URI, KiWiUriResource>() {
- @Override
- public KiWiUriResource load(URI key) throws Exception {
- return createURI(key.stringValue());
- }
- });
+ this.uriCache = new SelfPopulatingCache(cacheManager.getCache("uriCache"), new CacheEntryFactory() {
+ @Override
+ public Object createEntry(Object key) throws Exception {
+ return createURI(((URI)key).stringValue());
+ }
+ });
- this.bnodeCache = CacheBuilder.newBuilder()
- .recordStats()
- .maximumWeight(bnodeCacheSize)
- .weigher(new CacheWeigher<BNode, KiWiAnonResource>())
- .build(new CacheLoader<BNode, KiWiAnonResource>() {
- @Override
- public KiWiAnonResource load(BNode key) throws Exception {
- return createBNode(key.stringValue());
- }
- });
+ this.bnodeCache = new SelfPopulatingCache(cacheManager.getCache("bnodeCache"), new CacheEntryFactory() {
+ @Override
+ public Object createEntry(Object key) throws Exception {
+ return createBNode(((BNode)key).stringValue());
+ }
+ });
this.localeCache = CacheBuilder.newBuilder()
.maximumSize(100)
@@ -255,15 +280,26 @@ public class KiWiHandler implements RDFHandler {
} else if(value instanceof KiWiNode) {
return (KiWiNode)value;
} else if(value instanceof URI) {
- return uriCache.get((URI)value);
+ Element e = uriCache.get((URI) value);
+ if(e != null) {
+ return (KiWiNode) e.getObjectValue();
+ }
} else if(value instanceof BNode) {
- return bnodeCache.get(((BNode)value));
+ Element e = bnodeCache.get(((BNode)value));
+ if(e != null) {
+ return (KiWiNode) e.getObjectValue();
+ }
} else if(value instanceof Literal) {
Literal l = (Literal)value;
- return literalCache.get(l);
+ Element e = literalCache.get(l);
+ if(e != null) {
+ return (KiWiNode) e.getObjectValue();
+ }
+
} else {
throw new IllegalArgumentException("the value passed as argument does not have the correct type");
}
+ throw new IllegalStateException("could not construct or load node");
}
@@ -285,7 +321,7 @@ public class KiWiHandler implements RDFHandler {
KiWiLiteral result;
- final KiWiUriResource rtype = type==null ? null : uriCache.get(type);
+ final KiWiUriResource rtype = type==null ? null : (KiWiUriResource) convertNode(type);
try {
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/57865548/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/Statistics.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/Statistics.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/Statistics.java
index b02b8a2..f4a2be8 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/Statistics.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/Statistics.java
@@ -17,8 +17,7 @@
package org.apache.marmotta.kiwi.loader.generic;
-import com.google.common.cache.CacheStats;
-import com.google.common.cache.LoadingCache;
+import net.sf.ehcache.constructs.blocking.SelfPopulatingCache;
import org.apache.marmotta.kiwi.loader.util.UnitFormatter;
import org.rrd4j.ConsolFun;
import org.rrd4j.DsType;
@@ -157,10 +156,9 @@ public class Statistics {
public void run() {
long cacheMisses = 0, cacheHits = 0;
- for(LoadingCache c : new LoadingCache[] { handler.literalCache, handler.uriCache, handler.bnodeCache }) {
- CacheStats stats = c.stats();
- cacheHits += stats.hitCount();
- cacheMisses += stats.missCount();
+ for(SelfPopulatingCache c : new SelfPopulatingCache[] { handler.literalCache, handler.uriCache, handler.bnodeCache }) {
+ cacheHits += c.getStatistics().getCacheHits();
+ cacheMisses += c.getStatistics().getCacheMisses();
}
try {