You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@marmotta.apache.org by ss...@apache.org on 2013/11/21 13:00:51 UTC
git commit: move loader cache to the KiWi triplestore caches (towards
MARMOTTA-369)
Updated Branches:
refs/heads/develop 7bdf80d5a -> 0a3a562e1
move loader cache to the KiWi triplestore caches (towards MARMOTTA-369)
Project: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/commit/0a3a562e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/tree/0a3a562e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-marmotta/diff/0a3a562e
Branch: refs/heads/develop
Commit: 0a3a562e1393b2f3ab06033f33fa0068a40c5576
Parents: 7bdf80d
Author: Sebastian Schaffert <ss...@apache.org>
Authored: Thu Nov 21 13:00:48 2013 +0100
Committer: Sebastian Schaffert <ss...@apache.org>
Committed: Thu Nov 21 13:00:48 2013 +0100
----------------------------------------------------------------------
.../kiwi/loader/generic/KiWiHandler.java | 36 ++------------------
.../marmotta/kiwi/caching/KiWiCacheManager.java | 10 ++++++
.../src/main/resources/ehcache-kiwi.xml | 11 +++++-
3 files changed, 23 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/0a3a562e/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
index 6329662..f100791 100644
--- a/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
+++ b/libraries/kiwi/kiwi-loader/src/main/java/org/apache/marmotta/kiwi/loader/generic/KiWiHandler.java
@@ -3,14 +3,9 @@ package org.apache.marmotta.kiwi.loader.generic;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
-import net.sf.ehcache.Cache;
-import net.sf.ehcache.CacheManager;
import net.sf.ehcache.Element;
-import net.sf.ehcache.config.CacheConfiguration;
-import net.sf.ehcache.config.MemoryUnit;
import net.sf.ehcache.constructs.blocking.CacheEntryFactory;
import net.sf.ehcache.constructs.blocking.SelfPopulatingCache;
-import net.sf.ehcache.store.MemoryStoreEvictionPolicy;
import org.apache.marmotta.commons.sesame.model.Namespaces;
import org.apache.marmotta.commons.util.DateUtils;
import org.apache.marmotta.kiwi.loader.KiWiLoaderConfiguration;
@@ -46,9 +41,6 @@ import static org.apache.marmotta.kiwi.loader.util.UnitFormatter.formatSize;
*/
public class KiWiHandler implements RDFHandler {
- private static double CACHE_MEMORY_PERCENTAGE = 0.20;
-
-
private static Logger log = LoggerFactory.getLogger(KiWiHandler.class);
protected KiWiConnection connection;
@@ -71,49 +63,27 @@ public class KiWiHandler implements RDFHandler {
// if non-null, all imported statements will have this context (regardless whether they specified a different context)
private KiWiResource overrideContext;
- private CacheManager cacheManager;
-
private Statistics statistics;
public KiWiHandler(KiWiStore store, KiWiLoaderConfiguration config) {
this.config = config;
this.store = store;
- this.cacheManager = CacheManager.create();
-
-
- long usedMemory = Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory();
- long freeMemory = Runtime.getRuntime().maxMemory() - usedMemory;
- long cacheSize = (long) (freeMemory * CACHE_MEMORY_PERCENTAGE);
-
-
- log.info("calculated cache size: {}B", formatSize(cacheSize));
-
-
- Cache baseCache = new Cache(
- new CacheConfiguration("loader-cache",0)
- .maxBytesLocalHeap(cacheSize, MemoryUnit.BYTES)
- .memoryStoreEvictionPolicy(MemoryStoreEvictionPolicy.LRU)
- .timeToIdleSeconds(120)
- .statistics(true)
- );
- cacheManager.addCache(baseCache);
-
- this.literalCache = new SelfPopulatingCache(cacheManager.getCache("loader-cache"), new CacheEntryFactory() {
+ this.literalCache = new SelfPopulatingCache(store.getPersistence().getCacheManager().getLoaderCache(), new CacheEntryFactory() {
@Override
public Object createEntry(Object key) throws Exception {
return createLiteral((Literal)key);
}
});
- this.uriCache = new SelfPopulatingCache(cacheManager.getCache("loader-cache"), new CacheEntryFactory() {
+ this.uriCache = new SelfPopulatingCache(store.getPersistence().getCacheManager().getLoaderCache(), new CacheEntryFactory() {
@Override
public Object createEntry(Object key) throws Exception {
return createURI(((URI)key).stringValue());
}
});
- this.bnodeCache = new SelfPopulatingCache(cacheManager.getCache("loader-cache"), new CacheEntryFactory() {
+ this.bnodeCache = new SelfPopulatingCache(store.getPersistence().getCacheManager().getLoaderCache(), new CacheEntryFactory() {
@Override
public Object createEntry(Object key) throws Exception {
return createBNode(((BNode)key).stringValue());
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/0a3a562e/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java
index 86ac308..38be64c 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java
+++ b/libraries/kiwi/kiwi-triplestore/src/main/java/org/apache/marmotta/kiwi/caching/KiWiCacheManager.java
@@ -110,6 +110,16 @@ public class KiWiCacheManager {
return cacheManager.getCache("namespace-prefix-cache");
}
+
+ /**
+ * Return the cache used by the KiWiLoader. Used for mapping from Sesame nodes to KiWi nodes.
+ * @return
+ */
+ public Cache getLoaderCache() {
+ return cacheManager.getCache("loader-cache");
+ }
+
+
/**
* Get the cache with the given name from the cache manager. Can be used to request additional
* caches from the cache manager that are not covered by explicit methods.
http://git-wip-us.apache.org/repos/asf/incubator-marmotta/blob/0a3a562e/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml
----------------------------------------------------------------------
diff --git a/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml b/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml
index b18cb45..3c7fd17 100644
--- a/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml
+++ b/libraries/kiwi/kiwi-triplestore/src/main/resources/ehcache-kiwi.xml
@@ -46,7 +46,7 @@ are "on" and "off". The default is "autodetect".
<ehcache name="kiwi"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
updateCheck="false"
- maxBytesLocalHeap="50%"
+ maxBytesLocalHeap="60%"
xsi:noNamespaceSchemaLocation="http://ehcache.sf.net/ehcache.xsd">
<!--
@@ -209,6 +209,15 @@ are "on" and "off". The default is "autodetect".
overflowToDisk="false"
memoryStoreEvictionPolicy="LFU"/>
+
+ <cache name="loader-cache"
+ statistics="true"
+ maxBytesLocalHeap="15%"
+ timeToIdleSeconds="120"
+ overflowToDisk="false"
+ memoryStoreEvictionPolicy="LRU"/>
+
+
<!-- uncomment to enable cache debugging -->
<!--
<cacheManagerPeerListenerFactory