You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@directmemory.apache.org by to...@apache.org on 2012/06/12 14:03:06 UTC
svn commit: r1349283 - in
/incubator/directmemory/trunk/integrations/solr/src:
main/java/org/apache/directmemory/examples/solr/
main/java/org/apache/directmemory/solr/
test/java/org/apache/directmemory/examples/solr/
test/java/org/apache/directmemory/s...
Author: tommaso
Date: Tue Jun 12 12:03:05 2012
New Revision: 1349283
URL: http://svn.apache.org/viewvc?rev=1349283&view=rev
Log:
[DIRECTMEMORY-89] - moving the classes in o.a.dm.solr instead of o.a.dm.examples.solr
Added:
incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/
incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/SolrOffHeapCache.java
incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/package-info.java
incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/
incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapCacheTest.java
incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapIntegrationTest.java
Removed:
incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/examples/solr/
incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/examples/solr/
Modified:
incubator/directmemory/trunk/integrations/solr/src/test/resources/solr/config/solrconfig.xml
Added: incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/SolrOffHeapCache.java
URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/SolrOffHeapCache.java?rev=1349283&view=auto
==============================================================================
--- incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/SolrOffHeapCache.java (added)
+++ incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/SolrOffHeapCache.java Tue Jun 12 12:03:05 2012
@@ -0,0 +1,321 @@
+package org.apache.directmemory.solr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.directmemory.DirectMemory;
+import org.apache.directmemory.cache.CacheService;
+import org.apache.directmemory.measures.Monitor;
+import org.apache.directmemory.measures.Ram;
+import org.apache.directmemory.serialization.Serializer;
+import org.apache.directmemory.serialization.SerializerFactory;
+import org.apache.directmemory.serialization.SerializerNotFoundException;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.search.CacheRegenerator;
+import org.apache.solr.search.SolrCache;
+import org.apache.solr.search.SolrIndexSearcher;
+
+/**
+ * A {@link SolrCache} based on Apache DirectMemory
+ */
+public class SolrOffHeapCache<K, V>
+ implements SolrCache<K, V>
+{
+
+ private static class CumulativeStats
+ {
+ AtomicLong lookups = new AtomicLong();
+
+ AtomicLong hits = new AtomicLong();
+
+ AtomicLong inserts = new AtomicLong();
+
+ AtomicLong evictions = new AtomicLong();
+ }
+
+ private CumulativeStats stats;
+
+ private long lookups;
+
+ private long hits;
+
+ private long inserts;
+
+ private long evictions;
+
+ private long warmupTime = 0;
+
+ private String name;
+
+ private int autowarmCount;
+
+ private State state;
+
+ private CacheRegenerator regenerator;
+
+ private String description = "DM Cache";
+
+ private CacheService<K, V> cacheService;
+
+ public CacheService<K, V> getCacheService()
+ {
+ return cacheService;
+ }
+
+ @Override
+ public Object init( Map args, Object persistence, CacheRegenerator regenerator )
+ {
+
+
+ Object buffers = args.get( "buffers" );
+ String sizeStr = String.valueOf( args.get( "size" ) );
+ Integer capacity = Integer.parseInt( String.valueOf( args.get( "initialSize" ) ) );
+
+ cacheService = new DirectMemory<K, V>()
+ .setNumberOfBuffers( buffers != null ? Integer.valueOf( String.valueOf( buffers ) ) : 1 )
+ .setInitialCapacity( Ram.Mb( Double.valueOf( capacity ) / 512 ) )
+ .setSize( Ram.Mb( Double.valueOf( sizeStr ) / 512 ) )
+ .newCacheService();
+
+ String serializerClassName = (String) args.get( "serializerClassName" );
+ if ( serializerClassName != null )
+ {
+ Serializer serializer;
+ try
+ {
+ serializer = SerializerFactory.createNewSerializer( serializerClassName );
+ }
+ catch ( SerializerNotFoundException e )
+ {
+ log.warn( "Serializer not found for class " + serializerClassName +
+ ", falling back to the default serializer", e);
+ serializer = SerializerFactory.createNewSerializer();
+ }
+
+ cacheService.setSerializer( serializer );
+ }
+
+ state = State.CREATED;
+ this.regenerator = regenerator;
+ name = (String) args.get( "name" );
+ final int limit = sizeStr == null ? 1024 : Integer.parseInt( sizeStr );
+ String str = (String) args.get( "initialSize" );
+ final int initialSize = Math.min( str == null ? 1024 : Integer.parseInt( str ), limit );
+ str = (String) args.get( "autowarmCount" );
+ autowarmCount = str == null ? 0 : Integer.parseInt( str );
+
+ description = "Solr OffHeap Cache(maxSize=" + limit + ", initialSize=" + initialSize;
+ if ( autowarmCount > 0 )
+ {
+ description += ", autowarmCount=" + autowarmCount + ", regenerator=" + regenerator;
+ }
+ description += ')';
+
+ if ( persistence == null )
+ {
+ // must be the first time a cache of this type is being created
+ persistence = new CumulativeStats();
+ }
+
+ stats = (CumulativeStats) persistence;
+
+ return persistence;
+ }
+
+ @Override
+ public String name()
+ {
+ return name;
+ }
+
+ @Override
+ public int size()
+ {
+ return Long.valueOf( cacheService.entries() ).intValue();
+ }
+
+ @Override
+ public V put( K key, V value )
+ {
+ synchronized ( this )
+ {
+ if ( state == State.LIVE )
+ {
+ stats.inserts.incrementAndGet();
+ }
+
+ inserts++;
+ cacheService.put( key, value );
+ return value;
+ }
+ }
+
+ @Override
+ public V get( K key )
+ {
+ synchronized ( this )
+ {
+ V val = cacheService.retrieve( key );
+ if ( state == State.LIVE )
+ {
+ // only increment lookups and hits if we are live.
+ lookups++;
+ stats.lookups.incrementAndGet();
+ if ( val != null )
+ {
+ hits++;
+ stats.hits.incrementAndGet();
+ }
+ }
+ return val;
+ }
+ }
+
+ @Override
+ public void clear()
+ {
+ synchronized ( this )
+ {
+ cacheService.clear();
+ }
+ }
+
+ @Override
+ public void setState( State state )
+ {
+ this.state = state;
+ }
+
+ @Override
+ public State getState()
+ {
+ return state;
+ }
+
+ @Override
+ public void warm( SolrIndexSearcher searcher, SolrCache<K, V> old )
+ throws IOException
+ {
+ // it looks like there is no point in warming an off heap item
+ }
+
+ @Override
+ public void close()
+ {
+ cacheService.dump();
+ Monitor.dump();
+ }
+
+ @Override
+ public String getName()
+ {
+ return SolrOffHeapCache.class.getName();
+ }
+
+ @Override
+ public String getVersion()
+ {
+ return SolrCore.version;
+ }
+
+ public String getDescription()
+ {
+ return description;
+ }
+
+ public Category getCategory()
+ {
+ return Category.CACHE;
+ }
+
+ @Override
+ public String getSourceId()
+ {
+ return null;
+ }
+
+ @Override
+ public String getSource()
+ {
+ return null;
+ }
+
+ @Override
+ public URL[] getDocs()
+ {
+ return new URL[0];
+ }
+
+ public NamedList getStatistics()
+ {
+ NamedList lst = new SimpleOrderedMap();
+ synchronized ( this )
+ {
+ lst.add( "lookups", lookups );
+ lst.add( "hits", hits );
+ lst.add( "hitratio", calcHitRatio( lookups, hits ) );
+ lst.add( "inserts", inserts );
+ lst.add( "evictions", evictions );
+ lst.add( "size", cacheService.entries() );
+ }
+
+ lst.add( "warmupTime", warmupTime );
+
+ long clookups = stats.lookups.get();
+ long chits = stats.hits.get();
+ lst.add( "cumulative_lookups", clookups );
+ lst.add( "cumulative_hits", chits );
+ lst.add( "cumulative_hitratio", calcHitRatio( clookups, chits ) );
+ lst.add( "cumulative_inserts", stats.inserts.get() );
+ lst.add( "cumulative_evictions", stats.evictions.get() );
+
+ return lst;
+ }
+
+ private Object calcHitRatio( long clookups, long chits )
+ {
+ if ( lookups == 0 )
+ {
+ return "0.00";
+ }
+ if ( lookups == hits )
+ {
+ return "1.00";
+ }
+ int hundredths = (int) ( hits * 100 / lookups ); // rounded down
+ if ( hundredths < 10 )
+ {
+ return "0.0" + hundredths;
+ }
+ return "0." + hundredths;
+ }
+
+ @Override
+ public String toString()
+ {
+ return name + getStatistics().toString();
+ }
+}
Added: incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/package-info.java
URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/package-info.java?rev=1349283&view=auto
==============================================================================
--- incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/package-info.java (added)
+++ incubator/directmemory/trunk/integrations/solr/src/main/java/org/apache/directmemory/solr/package-info.java Tue Jun 12 12:03:05 2012
@@ -0,0 +1,23 @@
+/**
+ * Apache Solr cache integration.
+ */
+package org.apache.directmemory.solr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
Added: incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapCacheTest.java
URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapCacheTest.java?rev=1349283&view=auto
==============================================================================
--- incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapCacheTest.java (added)
+++ incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapCacheTest.java Tue Jun 12 12:03:05 2012
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.directmemory.solr;
+
+import org.apache.directmemory.serialization.StandardSerializer;
+import org.apache.directmemory.solr.SolrOffHeapCache;
+import org.apache.lucene.search.MatchAllDocsQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Sort;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.search.QueryResultKey;
+import org.apache.solr.search.function.DocValues;
+import org.apache.solr.search.function.QueryValueSource;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+/**
+ * Testcase for {@link org.apache.directmemory.solr.SolrOffHeapCache}
+ */
+public class SolrOffHeapCacheTest
+{
+
+ private Logger log = LoggerFactory.getLogger( getClass() );
+
+ private SolrOffHeapCache solrOffHeapCache;
+
+ @Before
+ public void setUp()
+ {
+ solrOffHeapCache = new SolrOffHeapCache();
+ Map<String, String> args = new HashMap<String, String>();
+ args.put( "size", "10000" );
+ args.put( "initialSize", "1000" );
+ args.put( "serializerClassName", StandardSerializer.class.getName() );
+ try
+ {
+ solrOffHeapCache.init( args, null, null );
+ }
+ catch ( NoClassDefFoundError e )
+ {
+ log.error( e.getMessage(), e );
+ throw e;
+ }
+ }
+
+ @After
+ public void tearDown()
+ {
+ solrOffHeapCache.clear();
+ solrOffHeapCache.close();
+ }
+
+ @Test
+ public void testStatisticsWhenCacheNotUsedYet()
+ throws Exception
+ {
+
+ NamedList stats = solrOffHeapCache.getStatistics();
+ assertNotNull( stats );
+ assertEquals( 0l, stats.get( "lookups" ) );
+ assertEquals( 0l, stats.get( "evictions" ) );
+ assertEquals( 0l, stats.get( "hits" ) );
+ assertEquals( 0l, stats.get( "inserts" ) );
+ }
+
+ @Test
+ public void testPut()
+ throws Exception
+ {
+
+ MatchAllDocsQuery query = new MatchAllDocsQuery();
+ QueryResultKey queryResultKey =
+ new QueryResultKey(query, new ArrayList<Query>(), new Sort(), 1 );
+
+ solrOffHeapCache.put( queryResultKey, new QueryValueSource(query,1) );
+
+ }
+}
Added: incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapIntegrationTest.java
URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapIntegrationTest.java?rev=1349283&view=auto
==============================================================================
--- incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapIntegrationTest.java (added)
+++ incubator/directmemory/trunk/integrations/solr/src/test/java/org/apache/directmemory/solr/SolrOffHeapIntegrationTest.java Tue Jun 12 12:03:05 2012
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.directmemory.solr;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.directmemory.solr.SolrOffHeapCache;
+import org.apache.solr.request.LocalSolrQueryRequest;
+import org.apache.solr.util.TestHarness;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ */
+public class SolrOffHeapIntegrationTest
+{
+
+ private static SolrOffHeapCache<Object, Object> solrOffHeapCache;
+
+ private static TestHarness h;
+
+ @BeforeClass
+ public static void setUp()
+ {
+ String data = "target/data/expand";
+ String config = SolrOffHeapCache.class.getResource( "/solr/config/solrconfig.xml" ).getFile();
+ String schema = SolrOffHeapCache.class.getResource( "/solr/config/schema.xml" ).getFile();
+ h = new TestHarness( data, config, schema );
+ solrOffHeapCache = new SolrOffHeapCache<Object, Object>();
+ }
+
+ @AfterClass
+ public static void tearDown()
+ {
+ // FIXME restore this when issue with static is fixed
+ //solrOffHeapCache.getCacheService().clear();
+ }
+
+ @Test
+ @Ignore // FIXME need TomNaso help - now I see why he needed a static CacheService reference
+ public void testSimpleQuery()
+ throws Exception
+ {
+
+ // add a doc to Solr
+ h.validateAddDoc( "id", "1", "text", "something is happening here" );
+
+ // make the query
+ LocalSolrQueryRequest request = h.getRequestFactory( "standard", 0, 10 ).makeRequest( "q", "something" );
+ String response = h.query( "standard", request );
+ assertTrue( response != null );
+ assertTrue( !response.contains( "error" ) );
+
+ // check the cache has been hit
+ assertTrue( solrOffHeapCache.getCacheService().entries() > 0 );
+
+
+ }
+}
Modified: incubator/directmemory/trunk/integrations/solr/src/test/resources/solr/config/solrconfig.xml
URL: http://svn.apache.org/viewvc/incubator/directmemory/trunk/integrations/solr/src/test/resources/solr/config/solrconfig.xml?rev=1349283&r1=1349282&r2=1349283&view=diff
==============================================================================
--- incubator/directmemory/trunk/integrations/solr/src/test/resources/solr/config/solrconfig.xml (original)
+++ incubator/directmemory/trunk/integrations/solr/src/test/resources/solr/config/solrconfig.xml Tue Jun 12 12:03:05 2012
@@ -360,7 +360,7 @@
for each document). Since Lucene internal document ids are
transient, this cache will not be autowarmed.
-->
- <queryResultCache class="org.apache.directmemory.examples.solr.SolrOffHeapCache" size="10000"
+ <queryResultCache class="org.apache.directmemory.solr.SolrOffHeapCache" size="10000"
initialSize="1000" autowarmCount="0" buffers="2"
serializerClassName="org.apache.directmemory.serialization.protostuff.ProtoStuffWithLinkedBufferSerializer"/>