You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2014/05/14 09:44:06 UTC
svn commit: r1594505 - in /lucene/dev/branches/lucene5666: lucene/
lucene/analysis/common/src/test/org/apache/lucene/collation/ lucene/core/
lucene/grouping/src/java/org/apache/lucene/search/grouping/
lucene/grouping/src/java/org/apache/lucene/search/g...
Author: rmuir
Date: Wed May 14 07:44:05 2014
New Revision: 1594505
URL: http://svn.apache.org/r1594505
Log:
LUCENE-5666: clear nocommits and fix precommit
Modified:
lucene/dev/branches/lucene5666/lucene/CHANGES.txt
lucene/dev/branches/lucene5666/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
lucene/dev/branches/lucene5666/lucene/core/build.xml
lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/package.html
lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html
lucene/dev/branches/lucene5666/lucene/misc/build.xml
lucene/dev/branches/lucene5666/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
lucene/dev/branches/lucene5666/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java
lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
Modified: lucene/dev/branches/lucene5666/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/CHANGES.txt?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene5666/lucene/CHANGES.txt Wed May 14 07:44:05 2014
@@ -65,6 +65,18 @@ API Changes
as tokens anymore, and now iterates cells on-demand during indexing instead of
building a collection. RPT now has more setters. (David Smiley)
+* LUCENE-5666: Change uninverted access (sorting, faceting, grouping, etc)
+ to use the DocValues API instead of FieldCache. For FieldCache functionality,
+ use UninvertingReader in lucene/misc (or implement your own FilterReader).
+ UninvertingReader is more efficient: supports multi-valued numeric fields,
+ detects when a multi-valued field is single-valued, reuses caches
+ of compatible types (e.g. SORTED also supports BINARY and SORTED_SET access
+ without insanity). "Insanity" is no longer possible unless you explicitly want it.
+ Rename FieldCache* and DocTermOrds* classes in the search package to DocValues*.
+ Move SortedSetSortField to core and add SortedSetFieldSource to queries/, which
+ takes the same selectors. Add helper methods to DocValues.java that are better
+ suited for search code (never return null, etc). (Mike McCandless, Robert Muir)
+
Documentation
* LUCENE-5392: Add/improve analysis package documentation to reflect
Modified: lucene/dev/branches/lucene5666/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java (original)
+++ lucene/dev/branches/lucene5666/lucene/analysis/common/src/test/org/apache/lucene/collation/TestCollationDocValuesField.java Wed May 14 07:44:05 2014
@@ -85,7 +85,7 @@ public class TestCollationDocValuesField
RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
Document doc = new Document();
Field field = newField("field", "", StringField.TYPE_STORED);
- Collator collator = Collator.getInstance(); // uses -Dtests.locale
+ Collator collator = Collator.getInstance(Locale.getDefault()); // uses -Dtests.locale
if (random().nextBoolean()) {
collator.setStrength(Collator.PRIMARY);
}
Modified: lucene/dev/branches/lucene5666/lucene/core/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/core/build.xml?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/core/build.xml (original)
+++ lucene/dev/branches/lucene5666/lucene/core/build.xml Wed May 14 07:44:05 2014
@@ -31,7 +31,6 @@
"/>
<property name="forbidden-rue-excludes" value="
- org/apache/lucene/search/FieldCache$CacheEntry.class
org/apache/lucene/util/RamUsageEstimator.class
org/apache/lucene/search/CachingWrapperFilter.class
"/>
Modified: lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (original)
+++ lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java Wed May 14 07:44:05 2014
@@ -300,7 +300,7 @@ public class BlockGroupingCollector exte
* This is normally not a problem, as you can obtain the
* value just like you obtain other values for each
* matching document (eg, via stored fields, via
- * FieldCache, etc.)
+ * DocValues, etc.)
*
* @param withinGroupSort The {@link Sort} used to sort
* documents within each group. Passing null is
Modified: lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/package.html?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/package.html (original)
+++ lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/package.html Wed May 14 07:44:05 2014
@@ -80,8 +80,7 @@ field fall into a single group.</p>
<p>Known limitations:</p>
<ul>
<li> For the two-pass grouping search, the group field must be a
- single-valued indexed field (or indexed as a {@link org.apache.lucene.document.SortedDocValuesField}).
- {@link org.apache.lucene.search.FieldCache} is used to load the {@link org.apache.lucene.index.SortedDocValues} for this field.
+ indexed as a {@link org.apache.lucene.document.SortedDocValuesField}).
<li> Although Solr support grouping by function and this module has abstraction of what a group is, there are currently only
implementations for grouping based on terms.
<li> Sharding is not directly supported, though is not too
Modified: lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java (original)
+++ lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/TermFirstPassGroupingCollector.java Wed May 14 07:44:05 2014
@@ -46,7 +46,7 @@ public class TermFirstPassGroupingCollec
*
* @param groupField The field used to group
* documents. This field must be single-valued and
- * indexed (FieldCache is used to access its value
+ * indexed (DocValues is used to access its value
* per-document).
* @param groupSort The {@link Sort} used to sort the
* groups. The top sorted document within each group
Modified: lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html (original)
+++ lucene/dev/branches/lucene5666/lucene/grouping/src/java/org/apache/lucene/search/grouping/term/package.html Wed May 14 07:44:05 2014
@@ -16,6 +16,6 @@
-->
<html>
<body>
-Support for grouping by indexed terms via {@link org.apache.lucene.search.FieldCache}.
+Support for grouping by indexed terms via {@link org.apache.lucene.index.DocValues}.
</body>
</html>
Modified: lucene/dev/branches/lucene5666/lucene/misc/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/misc/build.xml?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/misc/build.xml (original)
+++ lucene/dev/branches/lucene5666/lucene/misc/build.xml Wed May 14 07:44:05 2014
@@ -32,6 +32,10 @@
org/apache/lucene/misc/IndexMergeTool.class
"/>
+ <property name="forbidden-rue-excludes" value="
+ org/apache/lucene/uninverting/FieldCache$CacheEntry.class
+ "/>
+
<import file="../module-build.xml"/>
<target name="install-cpptasks" unless="cpptasks.uptodate" depends="ivy-availability-check,ivy-fail,ivy-configure">
Modified: lucene/dev/branches/lucene5666/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java (original)
+++ lucene/dev/branches/lucene5666/lucene/misc/src/java/org/apache/lucene/uninverting/UninvertingReader.java Wed May 14 07:44:05 2014
@@ -40,6 +40,7 @@ import org.apache.lucene.index.FilterDir
import org.apache.lucene.index.NumericDocValues;
import org.apache.lucene.index.SortedDocValues;
import org.apache.lucene.index.SortedSetDocValues;
+import org.apache.lucene.uninverting.FieldCache.CacheEntry;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.NumericUtils;
@@ -309,4 +310,17 @@ public class UninvertingReader extends F
public String toString() {
return "Uninverting(" + in.toString() + ")";
}
+
+ /**
+ * Return information about the backing cache
+ * @lucene.internal
+ */
+ public static String[] getUninvertedStats() {
+ CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
+ String[] info = new String[entries.length];
+ for (int i = 0; i < entries.length; i++) {
+ info[i] = entries[i].toString();
+ }
+ return info;
+ }
}
Modified: lucene/dev/branches/lucene5666/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java (original)
+++ lucene/dev/branches/lucene5666/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java Wed May 14 07:44:05 2014
@@ -19,28 +19,19 @@ package org.apache.solr.search;
import java.net.URL;
+import org.apache.lucene.uninverting.UninvertingReader;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.SolrCore;
import org.apache.solr.core.SolrInfoMBean;
-/*
-import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.FieldCache.CacheEntry;
-import org.apache.lucene.util.FieldCacheSanityChecker;
-import org.apache.lucene.util.FieldCacheSanityChecker.Insanity;
-*/
-// nocommit: maybe provide something useful here instead.
-
/**
* A SolrInfoMBean that provides introspection of the Solr FieldCache
*
*/
public class SolrFieldCacheMBean implements SolrInfoMBean {
- //protected FieldCacheSanityChecker checker = new FieldCacheSanityChecker();
-
@Override
public String getName() { return this.getClass().getName(); }
@Override
@@ -62,21 +53,11 @@ public class SolrFieldCacheMBean impleme
@Override
public NamedList getStatistics() {
NamedList stats = new SimpleOrderedMap();
- /*
- CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries();
+ String[] entries = UninvertingReader.getUninvertedStats();
stats.add("entries_count", entries.length);
for (int i = 0; i < entries.length; i++) {
- CacheEntry e = entries[i];
- stats.add("entry#" + i, e.toString());
+ stats.add("entry#" + i, entries[i]);
}
-
- Insanity[] insanity = checker.check(entries);
-
- stats.add("insanity_count", insanity.length);
- for (int i = 0; i < insanity.length; i++) {
-
- stats.add("insanity#" + i, insanity[i].toString());
- }*/
return stats;
}
Modified: lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java?rev=1594505&r1=1594504&r2=1594505&view=diff
==============================================================================
--- lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (original)
+++ lucene/dev/branches/lucene5666/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java Wed May 14 07:44:05 2014
@@ -417,10 +417,6 @@ public class TestFunctionQuery extends S
,"*//doc[1]/float[.='120.0']"
,"*//doc[2]/float[.='121.0']"
);
-
-
- // nocommit: split test if needed
- // FieldCache.DEFAULT.purgeAllCaches(); // hide FC insanity
}
/**