You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2012/09/06 10:02:37 UTC
svn commit: r1381504 - in /lucene/dev/trunk:
lucene/codecs/src/java/org/apache/lucene/codecs/bloom/
lucene/core/src/java/org/apache/lucene/util/
lucene/core/src/java/org/apache/lucene/util/hash/
lucene/core/src/resources/META-INF/services/ lucene/test-...
Author: jpountz
Date: Thu Sep 6 08:02:36 2012
New Revision: 1381504
URL: http://svn.apache.org/viewvc?rev=1381504&view=rev
Log:
LUCENE-4340: Move bloom PF utilities to lucene/codecs.
Added:
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java
- copied, changed from r1381222, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/FuzzySet.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java
- copied, changed from r1381222, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java
- copied, changed from r1381222, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/MurmurHash2.java
Removed:
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/FuzzySet.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/
lucene/dev/trunk/lucene/core/src/resources/META-INF/services/org.apache.lucene.util.hash.HashFunction
Modified:
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java
lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/bloom/TestBloomFilteredLucene40Postings.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java?rev=1381504&r1=1381503&r2=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilterFactory.java Thu Sep 6 08:02:36 2012
@@ -17,7 +17,6 @@ package org.apache.lucene.codecs.bloom;
*/
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.FuzzySet;
/**
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java?rev=1381504&r1=1381503&r2=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java Thu Sep 6 08:02:36 2012
@@ -33,6 +33,7 @@ import org.apache.lucene.codecs.Postings
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.TermStats;
import org.apache.lucene.codecs.TermsConsumer;
+import org.apache.lucene.codecs.bloom.FuzzySet.ContainsResult;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo;
@@ -46,11 +47,8 @@ import org.apache.lucene.store.IndexInpu
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.FuzzySet;
-import org.apache.lucene.util.FuzzySet.ContainsResult;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.automaton.CompiledAutomaton;
-import org.apache.lucene.util.hash.MurmurHash2;
/**
* <p>
Modified: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java?rev=1381504&r1=1381503&r2=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/DefaultBloomFilterFactory.java Thu Sep 6 08:02:36 2012
@@ -17,9 +17,6 @@ package org.apache.lucene.codecs.bloom;
*/
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.FuzzySet;
-import org.apache.lucene.util.hash.HashFunction;
-import org.apache.lucene.util.hash.MurmurHash2;
/**
* Default policy is to allocate a bitset with 10% saturation given a unique term per document.
Copied: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java (from r1381222, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/FuzzySet.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java?p2=lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/FuzzySet.java&r1=1381222&r2=1381504&rev=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/FuzzySet.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/FuzzySet.java Thu Sep 6 08:02:36 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.util;
+package org.apache.lucene.codecs.bloom;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -20,7 +20,8 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.util.hash.HashFunction;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.FixedBitSet;
/**
* <p>
@@ -45,9 +46,20 @@ import org.apache.lucene.util.hash.HashF
* @lucene.experimental
*/
public class FuzzySet {
-
- public static final int FUZZY_SERIALIZATION_VERSION=1;
-
+
+ public static final int VERSION_SPI = 1; // HashFunction used to be loaded through a SPI
+ public static final int VERSION_START = VERSION_SPI;
+ public static final int VERSION_CURRENT = 2;
+
+ public static HashFunction hashFunctionForVersion(int version) {
+ if (version < VERSION_START) {
+ throw new IllegalArgumentException("Version " + version + " is too old, expected at least " + VERSION_START);
+ } else if (version > VERSION_CURRENT) {
+ throw new IllegalArgumentException("Version " + version + " is too new, expected at most " + VERSION_CURRENT);
+ }
+ return MurmurHash2.INSTANCE;
+ }
+
/**
* Result from {@link FuzzySet#contains(BytesRef)}:
* can never return definitively YES (always MAYBE),
@@ -174,8 +186,7 @@ public class FuzzySet {
*/
public void serialize(DataOutput out) throws IOException
{
- out.writeInt(FUZZY_SERIALIZATION_VERSION);
- out.writeString(hashFunction.getName());
+ out.writeInt(VERSION_CURRENT);
out.writeInt(bloomSize);
long[] bits = filter.getBits();
out.writeInt(bits.length);
@@ -188,11 +199,10 @@ public class FuzzySet {
public static FuzzySet deserialize(DataInput in) throws IOException
{
int version=in.readInt();
- if(version!=FUZZY_SERIALIZATION_VERSION)
- {
- throw new IOException("Error deserializing: set version is not "+FUZZY_SERIALIZATION_VERSION);
+ if (version == VERSION_SPI) {
+ in.readString();
}
- HashFunction hashFunction=HashFunction.forName(in.readString());
+ final HashFunction hashFunction = hashFunctionForVersion(version);
int bloomSize=in.readInt();
int numLongs=in.readInt();
long[]longs=new long[numLongs];
Copied: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java (from r1381222, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java?p2=lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java&r1=1381222&r2=1381504&rev=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/HashFunction.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/HashFunction.java Thu Sep 6 08:02:36 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.util.hash;
+package org.apache.lucene.codecs.bloom;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,10 +15,7 @@ package org.apache.lucene.util.hash;
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import java.util.Set;
-
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.NamedSPILoader;
/**
@@ -27,7 +24,7 @@ import org.apache.lucene.util.NamedSPILo
* on the range of bytes referenced in the provided {@link BytesRef}
* @lucene.experimental
*/
-public abstract class HashFunction implements NamedSPILoader.NamedSPI {
+public abstract class HashFunction {
/**
* Hashes the contents of the referenced bytes
@@ -35,50 +32,5 @@ public abstract class HashFunction imple
* @return the hash of the bytes referenced by bytes.offset and length bytes.length
*/
public abstract int hash(BytesRef bytes);
-
- private static final NamedSPILoader<HashFunction> loader =
- new NamedSPILoader<HashFunction>(HashFunction.class);
-
- private final String name;
- public HashFunction(String name) {
- NamedSPILoader.checkServiceName(name);
- this.name = name;
- }
-
- /** Returns this codec's name */
- @Override
- public final String getName() {
- return name;
- }
-
- /** looks up a hash function by name */
- public static HashFunction forName(String name) {
- return loader.lookup(name);
- }
-
- /** returns a list of all available hash function names */
- public static Set<String> availableHashFunctionNames() {
- return loader.availableServices();
- }
-
- /**
- * Reloads the hash function list from the given {@link ClassLoader}.
- * Changes to the function list are visible after the method ends, all
- * iterators ({@link #availableHashFunctionNames()},...) stay consistent.
- *
- * <p><b>NOTE:</b> Only new functions are added, existing ones are
- * never removed or replaced.
- *
- * <p><em>This method is expensive and should only be called for discovery
- * of new functions on the given classpath/classloader!</em>
- */
- public static void reloadHashFunctions(ClassLoader classloader) {
- loader.reload(classloader);
- }
-
- @Override
- public String toString() {
- return name;
- }
}
Copied: lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java (from r1381222, lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/MurmurHash2.java)
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java?p2=lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java&p1=lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/MurmurHash2.java&r1=1381222&r2=1381504&rev=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/hash/MurmurHash2.java (original)
+++ lucene/dev/trunk/lucene/codecs/src/java/org/apache/lucene/codecs/bloom/MurmurHash2.java Thu Sep 6 08:02:36 2012
@@ -1,4 +1,4 @@
-package org.apache.lucene.util.hash;
+package org.apache.lucene.codecs.bloom;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -32,14 +32,11 @@ import org.apache.lucene.util.BytesRef;
* </p>
* @lucene.experimental
*/
-public class MurmurHash2 extends HashFunction{
-
-
- public static final String HASH_NAME="MurmurHash2";
-
- public MurmurHash2() {
- super(HASH_NAME);
- }
+public final class MurmurHash2 extends HashFunction{
+
+ public static final MurmurHash2 INSTANCE = new MurmurHash2();
+
+ private MurmurHash2() {}
public static int hash(byte[] data, int seed, int offset, int len) {
int m = 0x5bd1e995;
Modified: lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/bloom/TestBloomFilteredLucene40Postings.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/bloom/TestBloomFilteredLucene40Postings.java?rev=1381504&r1=1381503&r2=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/bloom/TestBloomFilteredLucene40Postings.java (original)
+++ lucene/dev/trunk/lucene/test-framework/src/java/org/apache/lucene/codecs/bloom/TestBloomFilteredLucene40Postings.java Thu Sep 6 08:02:36 2012
@@ -22,13 +22,10 @@ import java.io.IOException;
import org.apache.lucene.codecs.FieldsConsumer;
import org.apache.lucene.codecs.FieldsProducer;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.codecs.bloom.BloomFilteringPostingsFormat;
import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat;
import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
-import org.apache.lucene.util.FuzzySet;
-import org.apache.lucene.util.hash.MurmurHash2;
/**
* A class used for testing {@link BloomFilteringPostingsFormat} with a concrete
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=1381504&r1=1381503&r2=1381504&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrResourceLoader.java Thu Sep 6 08:02:36 2012
@@ -36,7 +36,6 @@ import org.apache.lucene.analysis.util.T
import org.apache.lucene.analysis.util.TokenizerFactory;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.PostingsFormat;
-import org.apache.lucene.util.hash.HashFunction;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.solr.common.ResourceLoader;
import org.apache.solr.handler.admin.CoreAdminHandler;
@@ -176,8 +175,6 @@ public class SolrResourceLoader implemen
* this ResourceLoader.
*/
void reloadLuceneSPI() {
- // Hash functions:
- HashFunction.reloadHashFunctions(this.classLoader);
// Codecs:
PostingsFormat.reloadPostingsFormats(this.classLoader);
Codec.reloadCodecs(this.classLoader);