You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2011/04/04 15:03:48 UTC
svn commit: r1088606 [4/5] - in /lucene/dev/branches/realtime_search: ./
dev-tools/eclipse/ dev-tools/idea/solr/ dev-tools/maven/
dev-tools/maven/solr/ dev-tools/maven/solr/contrib/analysis-extras/
dev-tools/maven/solr/contrib/clustering/ dev-tools/mav...
Modified: lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestICUCollationKeyFilterFactory.java Mon Apr 4 13:03:42 2011
@@ -34,6 +34,7 @@ import com.ibm.icu.text.Collator;
import com.ibm.icu.text.RuleBasedCollator;
import com.ibm.icu.util.ULocale;
+@Deprecated
public class TestICUCollationKeyFilterFactory extends BaseTokenTestCase {
/*
Modified: lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/clustering/build.xml Mon Apr 4 13:03:42 2011
@@ -41,8 +41,8 @@
<pathelement path="${dest}/classes"/>
<pathelement path="${dest}/test-classes"/>
<pathelement path="${java.class.path}"/>
- <pathelement location="${common-solr.dir}/build/tests"/> <!-- include solr test code -->
- <pathelement location="${common-solr.dir}/../lucene/build/classes/test-framework" /> <!-- include some lucene test code -->
+ <pathelement location="${solr-path}/build/test-framework"/> <!-- include solr test code -->
+ <pathelement location="${solr-path}/../lucene/build/classes/test-framework" /> <!-- include some lucene test code -->
<path refid="common.classpath"/>
<!-- DistributedClusteringComponentTest uses Jetty -->
<fileset dir="${solr-path}/example/lib">
@@ -64,7 +64,7 @@
<mkdir dir="${dest}/classes"/>
<mkdir dir="${build.javadoc}"/>
- <subant target="compileTests">
+ <subant target="compileTestFramework">
<fileset dir="${solr-path}" includes="build.xml"/>
</subant>
<subant target="make-manifest">
Modified: lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/build.xml?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/dataimporthandler/build.xml Mon Apr 4 13:03:42 2011
@@ -55,7 +55,7 @@
<path refid="classpath.jetty" />
<pathelement path="target/classes" />
<pathelement path="target/test-classes" />
- <pathelement location="${solr-path}/build/tests"/> <!-- include solr test code -->
+ <pathelement location="${solr-path}/build/test-framework"/> <!-- include solr test code -->
<pathelement location="${solr-path}/../lucene/build/classes/test-framework" /> <!-- include some lucene test code -->
<pathelement path="${java.class.path}"/>
</path>
@@ -67,7 +67,7 @@
<pathelement path="target/extras/classes" />
<pathelement path="target/test-classes" />
<pathelement path="target/extras/test-classes" />
- <pathelement location="${solr-path}/build/tests"/> <!-- include solr test code -->
+ <pathelement location="${solr-path}/build/test-framework"/> <!-- include solr test code -->
<pathelement location="${solr-path}/../lucene/build/classes/test-framework" /> <!-- include some lucene test code -->
<pathelement path="${java.class.path}"/>
</path>
@@ -89,7 +89,7 @@
<target name="init">
<mkdir dir="target/classes"/>
<mkdir dir="${build.javadoc}" />
- <subant target="compileTests">
+ <subant target="compileTestFramework">
<fileset dir="${solr-path}" includes="build.xml"/>
</subant>
<subant target="make-manifest">
Modified: lucene/dev/branches/realtime_search/solr/contrib/extraction/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/extraction/build.xml?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/extraction/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/extraction/build.xml Mon Apr 4 13:03:42 2011
@@ -39,7 +39,7 @@
<path refid="common.classpath" />
<pathelement path="${dest}/classes" />
<pathelement path="${dest}/test-classes" />
- <pathelement location="${solr-path}/build/tests"/> <!-- include solr test code -->
+ <pathelement location="${solr-path}/build/test-framework"/> <!-- include solr test code -->
<pathelement location="${solr-path}/../lucene/build/classes/test-framework" /> <!-- include some lucene test code -->
<pathelement path="${java.class.path}"/>
</path>
@@ -51,7 +51,7 @@
<target name="init">
<mkdir dir="${dest}/classes"/>
<mkdir dir="${build.javadoc}" />
- <subant target="compileTests">
+ <subant target="compileTestFramework">
<fileset dir="${solr-path}" includes="build.xml"/>
</subant>
<subant target="make-manifest">
Modified: lucene/dev/branches/realtime_search/solr/contrib/uima/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/contrib/uima/build.xml?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/contrib/uima/build.xml (original)
+++ lucene/dev/branches/realtime_search/solr/contrib/uima/build.xml Mon Apr 4 13:03:42 2011
@@ -40,7 +40,7 @@
<path refid="common.classpath" />
<pathelement path="${dest}/classes" />
<pathelement path="${dest}/test-classes" />
- <pathelement location="${solr-path}/build/tests"/> <!-- include solr test code -->
+ <pathelement location="${solr-path}/build/test-framework"/> <!-- include solr test code -->
<pathelement location="${solr-path}/../lucene/build/classes/test-framework" /> <!-- include some lucene test code -->
<pathelement path="${java.class.path}"/>
</path>
@@ -52,7 +52,7 @@
<target name="init">
<mkdir dir="${dest}/classes"/>
<mkdir dir="${build.javadoc}" />
- <subant target="compileTests">
+ <subant target="compileTestFramework">
<fileset dir="${solr-path}" includes="build.xml"/>
</subant>
<subant target="make-manifest">
Modified: lucene/dev/branches/realtime_search/solr/site/features.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/features.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/features.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/features.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
Modified: lucene/dev/branches/realtime_search/solr/site/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/index.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/index.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/index.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
@@ -229,6 +232,9 @@ document.write("Last Published: " + docu
<a href="#news">News</a>
<ul class="minitoc">
<li>
+<a href="#March+2011+-+Solr+3.1+Released">March 2011 - Solr 3.1 Released</a>
+</li>
+<li>
<a href="#25+June+2010+-+Solr+1.4.1+Released">25 June 2010 - Solr 1.4.1 Released</a>
</li>
<li>
@@ -343,7 +349,55 @@ customization is required.
<a name="N10059"></a><a name="news"></a>
<h2 class="boxed">News</h2>
<div class="section">
-<a name="N1005F"></a><a name="25+June+2010+-+Solr+1.4.1+Released"></a>
+<a name="N1005F"></a><a name="March+2011+-+Solr+3.1+Released"></a>
+<h3 class="boxed">March 2011 - Solr 3.1 Released</h3>
+<p>The Lucene PMC is pleased to announce the release of <a href="http://www.apache.org/dyn/closer.cgi/lucene/solr">Apache Solr 3.1</a>!
+ </p>
+<p>
+ Solr's version number was synced with Lucene following the Lucene/Solr merge, so Solr 3.1 contains Lucene 3.1.
+ Solr 3.1 is the first release after Solr 1.4.1.
+ </p>
+<p>
+ Solr 3.1 release highlights include
+ </p>
+<ul>
+
+<li>
+<a href="http://wiki.apache.org/solr/SimpleFacetParameters#Facet_by_Range">Numeric range facets</a> (similar to date faceting).</li>
+
+<li>New <a href="http://wiki.apache.org/solr/SpatialSearch">spatial search</a>, including spatial filtering, boosting and sorting capabilities.</li>
+
+<li>Example Velocity driven search UI at <a href="http://localhost:8983/solr/browse">http://localhost:8983/solr/browse</a>
+</li>
+
+<li>A new faster <a href="http://wiki.apache.org/solr/HighlightingParameters">termvector-based highlighter</a>.</li>
+
+<li>Extended dismax (edismax) query parser with support for fielded queries, enhanced relevancy, and full lucene syntax support.</li>
+
+<li>Distributed search support for the <a href="http://wiki.apache.org/solr/SpellCheckComponent">Spell check</a>
+ and <a href="http://wiki.apache.org/solr/TermsComponent">Terms</a> components.</li>
+
+<li>
+<a href="http://wiki.apache.org/solr/Suggester">Suggester</a>, a fast trie-based autocomplete component.</li>
+
+<li>Sort results by any <a href="http://wiki.apache.org/solr/FunctionQuery#Sort_By_Function any function">function query</a>.</li>
+
+<li>
+<a href="http://wiki.apache.org/solr/UpdateJSON">JSON document indexing</a>.</li>
+
+<li>
+<a href="http://wiki.apache.org/solr/CSVResponseWriter">CSV response format</a>
+</li>
+
+<li>
+<a href="http://wiki.apache.org/solr/SolrUIMA">Apache UIMA integration</a> for metadata extraction.</li>
+
+<li>Tons of optimizations, bugfixes, and new analysis capabilities via <a href="http://lucene.apache.org/java/docs/index.html">Apache Lucene 3.1</a>.</li>
+
+</ul>
+<p>See the <a href="http://svn.apache.org/repos/asf/lucene/dev/tags/lucene_solr_3_1/solr/CHANGES.txt">release notes</a> for a more complete list of all the new features, improvements, and bugfixes.
+ </p>
+<a name="N100CA"></a><a name="25+June+2010+-+Solr+1.4.1+Released"></a>
<h3 class="boxed">25 June 2010 - Solr 1.4.1 Released</h3>
<p>
Solr 1.4.1 has been released and is now available for public
@@ -356,7 +410,7 @@ customization is required.
See the <a href="http://svn.apache.org/repos/asf/lucene/solr/tags/release-1.4.1/CHANGES.txt">release notes</a>
for more details.
</p>
-<a name="N10074"></a><a name="7+May+2010+-+Apache+Lucene+Eurocon+2010+Coming+to+Prague+May+18-21"></a>
+<a name="N100DF"></a><a name="7+May+2010+-+Apache+Lucene+Eurocon+2010+Coming+to+Prague+May+18-21"></a>
<h3 class="boxed">7 May 2010 - Apache Lucene Eurocon 2010 Coming to Prague May 18-21</h3>
<p>
On May 18th to the 21st Prague will play host to the first
@@ -428,7 +482,7 @@ customization is required.
</li>
</ul>
-<a name="N100C0"></a><a name="10+November+2009+-+Solr+1.4+Released"></a>
+<a name="N1012B"></a><a name="10+November+2009+-+Solr+1.4+Released"></a>
<h3 class="boxed">10 November 2009 - Solr 1.4 Released</h3>
<p>
Solr 1.4 has been released and is now available for public <a href="http://www.apache.org/dyn/closer.cgi/lucene/solr/">download</a>!
@@ -460,7 +514,7 @@ customization is required.
</ul>
<p>See the <a href="http://svn.apache.org/repos/asf/lucene/solr/tags/release-1.4.0/CHANGES.txt">release notes</a> for more details.
</p>
-<a name="N100F0"></a><a name="20+August+2009+-+Solr%27s+first+book+is+published%21"></a>
+<a name="N1015B"></a><a name="20+August+2009+-+Solr%27s+first+book+is+published%21"></a>
<h3 class="boxed">20 August 2009 - Solr's first book is published!</h3>
<p>
@@ -475,7 +529,7 @@ customization is required.
</p>
<p>Finally, this book covers various deployment considerations to include indexing strategies and performance-oriented configuration that will enable you to scale Solr to meet the needs of a high-volume site.
</p>
-<a name="N10110"></a><a name="18+August+2009+-+Lucene+at+US+ApacheCon"></a>
+<a name="N1017B"></a><a name="18+August+2009+-+Lucene+at+US+ApacheCon"></a>
<h3 class="boxed">18 August 2009 - Lucene at US ApacheCon</h3>
<p>
@@ -551,7 +605,7 @@ Be sure not to miss:
Search</a> - Jason Rutherglen @ 15:00</li>
</ul>
-<a name="N10184"></a><a name="09+February+2009+-+Lucene+at+ApacheCon+Europe+2009+in+Amsterdam"></a>
+<a name="N101EF"></a><a name="09+February+2009+-+Lucene+at+ApacheCon+Europe+2009+in+Amsterdam"></a>
<h3 class="boxed">09 February 2009 - Lucene at ApacheCon Europe 2009 in Amsterdam</h3>
<p>
@@ -589,23 +643,23 @@ Be sure not to miss:
</ul>
-<a name="N101CD"></a><a name="19+December+2008+-+Solr+Logo+Contest+Results"></a>
+<a name="N10238"></a><a name="19+December+2008+-+Solr+Logo+Contest+Results"></a>
<h3 class="boxed">19 December 2008 - Solr Logo Contest Results</h3>
<p>Many great logos were submitted, but only one could be chosen. Congratulations Michiel,
the creator of the winning logo that is proudly displayed at the top of this page.
</p>
-<a name="N101D6"></a><a name="03+October+2008+-+Solr+Logo+Contest"></a>
+<a name="N10241"></a><a name="03+October+2008+-+Solr+Logo+Contest"></a>
<h3 class="boxed">03 October 2008 - Solr Logo Contest</h3>
<p>By popular demand, Solr is holding a contest to pick a new Solr logo. Details about how to submit an entry can be found <a href="http://wiki.apache.org/solr/LogoContest">on the wiki</a>. The Deadline for submissions is November 20th, 2008 @ 11:59PM GMT.
</p>
-<a name="N101E3"></a><a name="15+September+2008+-+Solr+1.3.0+Available"></a>
+<a name="N1024E"></a><a name="15+September+2008+-+Solr+1.3.0+Available"></a>
<h3 class="boxed">15 September 2008 - Solr 1.3.0 Available</h3>
<p>Solr 1.3.0 is available for public download. This version contains many enhancements and bug fixes, including distributed search capabilities,
Lucene 2.3.x performance improvements and many others.
</p>
<p>See the <a href="http://svn.apache.org/repos/asf/lucene/solr/tags/release-1.3.0/CHANGES.txt">release notes</a> for more details. Download is
available from a <a href="http://www.apache.org/dyn/closer.cgi/lucene/solr/">Apache Mirror</a>.</p>
-<a name="N101F7"></a><a name="28+August+2008+-+Lucene%2FSolr+at+ApacheCon+New+Orleans"></a>
+<a name="N10262"></a><a name="28+August+2008+-+Lucene%2FSolr+at+ApacheCon+New+Orleans"></a>
<h3 class="boxed">28 August 2008 - Lucene/Solr at ApacheCon New Orleans</h3>
<p>
@@ -627,7 +681,7 @@ Be sure not to miss:
<li>An <a href="http://us.apachecon.com/c/acus2008/schedule/2008/11/05">entire day of Lucene sessions</a> on November 5th</li>
</ul>
-<a name="N10225"></a><a name="03+September+2007+-+Lucene+at+ApacheCon+Atlanta"></a>
+<a name="N10290"></a><a name="03+September+2007+-+Lucene+at+ApacheCon+Atlanta"></a>
<h3 class="boxed">03 September 2007 - Lucene at ApacheCon Atlanta</h3>
<p>
<a href="http://www.us.apachecon.com"><img alt="ApacheCon US logo" class="float-right" src="http://www.apache.org/ads/ApacheCon/2007-usa-125x125.png"></a>
@@ -647,7 +701,7 @@ Be sure not to miss:
<li>November 16, 4:00 pm: <a href="http://us.apachecon.com/us2007/program/talk/2017"> Advanced Indexing Techniques with Apache Lucene</a> by Michael Busch. Information on payloads and advanced indexing techniques.</li>
</ul>
-<a name="N1025E"></a><a name="06+June+2007%3A+Release+1.2+available"></a>
+<a name="N102C9"></a><a name="06+June+2007%3A+Release+1.2+available"></a>
<h3 class="boxed">06 June 2007: Release 1.2 available</h3>
<p>
This is the first release since Solr graduated from the Incubator,
@@ -657,40 +711,40 @@ Be sure not to miss:
and more flexible plugins.
</p>
<p>See the <a href="http://svn.apache.org/repos/asf/lucene/solr/tags/release-1.2.0/CHANGES.txt">release notes</a> for more details.</p>
-<a name="N1026F"></a><a name="17+January+2007%3A+Solr+graduates+from+Incubator"></a>
+<a name="N102DA"></a><a name="17+January+2007%3A+Solr+graduates+from+Incubator"></a>
<h3 class="boxed">17 January 2007: Solr graduates from Incubator</h3>
<p>
Solr has graduated from the Apache Incubator, and is now a sub-project of Lucene.
</p>
-<a name="N10279"></a><a name="22+December+2006%3A+Release+1.1.0+available"></a>
+<a name="N102E4"></a><a name="22+December+2006%3A+Release+1.1.0+available"></a>
<h3 class="boxed">22 December 2006: Release 1.1.0 available</h3>
<p>
This is the first release since Solr joined the Incubator, and brings
many new features and performance optimizations including highlighting,
faceted search, and JSON/Python/Ruby response formats.
</p>
-<a name="N10283"></a><a name="15+August+2006%3A+Solr+at+ApacheCon+US"></a>
+<a name="N102EE"></a><a name="15+August+2006%3A+Solr+at+ApacheCon+US"></a>
<h3 class="boxed">15 August 2006: Solr at ApacheCon US</h3>
<p>Chris Hostetter will be presenting
<strong><a href="http://www.apachecon.com/2006/US/html/sessions.html#FR26">"Faceted Searching With Apache Solr"</a></strong>
at ApacheCon US 2006, on October 13th at 4:30pm.
See the <a href="http://www.us.apachecon.com/">ApacheCon</a> website for more details.
</p>
-<a name="N10296"></a><a name="21+April+2006%3A+Solr+at+ApacheCon"></a>
+<a name="N10301"></a><a name="21+April+2006%3A+Solr+at+ApacheCon"></a>
<h3 class="boxed">21 April 2006: Solr at ApacheCon</h3>
<p>Yonik Seeley will be presenting
<strong>"Apache Solr, a Full-Text Search Server based on Lucene"</strong>
at ApacheCon Europe 2006, on June 29th at 5:30pm.
See the <a href="http://www.eu.apachecon.com/">ApacheCon</a> website for more details.
</p>
-<a name="N102A7"></a><a name="21+February+2006%3A+nightly+builds"></a>
+<a name="N10312"></a><a name="21+February+2006%3A+nightly+builds"></a>
<h3 class="boxed">21 February 2006: nightly builds</h3>
<p>Solr now has nightly builds. This automatically creates a
<a href="http://people.apache.org/builds/lucene/solr/nightly/">downloadable version of Solr every
night</a>. All unit tests must pass, or a message is sent to
the developers mailing list and no new version is created. This
also updates the <a href="api/index.html">javadoc</a>.</p>
-<a name="N102B9"></a><a name="17+January+2006%3A+Solr+Joins+Apache+Incubator"></a>
+<a name="N10324"></a><a name="17+January+2006%3A+Solr+Joins+Apache+Incubator"></a>
<h3 class="boxed">17 January 2006: Solr Joins Apache Incubator</h3>
<p>Solr, a search server based on Lucene, has been accepted into the Apache Incubator.
Solr was originally developed by CNET Networks, and is widely used within CNET
Modified: lucene/dev/branches/realtime_search/solr/site/index.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/index.pdf?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/realtime_search/solr/site/issue_tracking.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/issue_tracking.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/issue_tracking.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/issue_tracking.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
Modified: lucene/dev/branches/realtime_search/solr/site/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/linkmap.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/linkmap.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/linkmap.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
@@ -285,6 +288,12 @@ document.write("Last Published: " + docu
<a href="api/solrj/index.html">SolrJ Javadoc</a> ___________________ <em>apidocs</em>
</li>
</ul>
+
+<ul>
+<li>
+<a href="api/test-framework/index.html">Test Framework Javadoc</a> ___________________ <em>apidocs</em>
+</li>
+</ul>
</ul>
</ul>
Modified: lucene/dev/branches/realtime_search/solr/site/linkmap.pdf
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/linkmap.pdf?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/realtime_search/solr/site/mailing_lists.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/mailing_lists.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/mailing_lists.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/mailing_lists.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
Modified: lucene/dev/branches/realtime_search/solr/site/skin/basic.css
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/skin/basic.css?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/skin/basic.css (original)
+++ lucene/dev/branches/realtime_search/solr/site/skin/basic.css Mon Apr 4 13:03:42 2011
@@ -163,4 +163,4 @@ p {
.codefrag {
font-family: "Courier New", Courier, monospace;
font-size: 110%;
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/realtime_search/solr/site/skin/print.css
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/skin/print.css?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/skin/print.css (original)
+++ lucene/dev/branches/realtime_search/solr/site/skin/print.css Mon Apr 4 13:03:42 2011
@@ -51,4 +51,4 @@ a:link, a:visited {
acronym {
border: 0;
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/realtime_search/solr/site/skin/profile.css
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/skin/profile.css?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/skin/profile.css (original)
+++ lucene/dev/branches/realtime_search/solr/site/skin/profile.css Mon Apr 4 13:03:42 2011
@@ -172,4 +172,4 @@ a:hover { color:#6587ff}
}
-
\ No newline at end of file
+
Modified: lucene/dev/branches/realtime_search/solr/site/skin/screen.css
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/skin/screen.css?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/skin/screen.css (original)
+++ lucene/dev/branches/realtime_search/solr/site/skin/screen.css Mon Apr 4 13:03:42 2011
@@ -584,4 +584,4 @@ p.instruction {
list-style-image: url('../images/instruction_arrow.png');
list-style-position: outside;
margin-left: 2em;
-}
\ No newline at end of file
+}
Modified: lucene/dev/branches/realtime_search/solr/site/tutorial.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/tutorial.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/tutorial.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/tutorial.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_1.3', 'skin/')" id="menu_1.3Title" class="menutitle">Resources</div>
<div id="menu_1.3" class="menuitemgroup">
Modified: lucene/dev/branches/realtime_search/solr/site/version_control.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/site/version_control.html?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/site/version_control.html (original)
+++ lucene/dev/branches/realtime_search/solr/site/version_control.html Mon Apr 4 13:03:42 2011
@@ -130,6 +130,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="api/solrj/index.html">SolrJ Javadoc</a>
</div>
+<div class="menuitem">
+<a href="api/test-framework/index.html">Test Framework Javadoc</a>
+</div>
</div>
<div onclick="SwitchMenu('menu_selected_1.3', 'skin/')" id="menu_selected_1.3Title" class="menutitle" style="background-image: url('skin/images/chapter_open.gif');">Resources</div>
<div id="menu_selected_1.3" class="selectedmenuitemgroup" style="display: block;">
Modified: lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/NamedList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/NamedList.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/NamedList.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/common/org/apache/solr/common/util/NamedList.java Mon Apr 4 13:03:42 2011
@@ -43,7 +43,7 @@ import java.io.Serializable;
* by key, so ResponseWriters that output to a format such as JSON will normally
* choose a data structure that allows order to be easily preserved in various
* clients (i.e. not a straight map).
- * If access by key is more important, see {@link SimpleOrderedMap},
+ * If access by key is more important for serialization, see {@link SimpleOrderedMap},
* or simply use a regular {@link Map}
* </p>
*
@@ -200,10 +200,14 @@ public class NamedList<T> implements Clo
/**
* Gets the value for the first instance of the specified name
* found.
- *
+ * <p>
+ * NOTE: this runs in linear time (it scans starting at the
+ * beginning of the list until it finds the first pair with
+ * the specified name).
* @return null if not found or if the value stored was null.
* @see #indexOf
* @see #get(String,int)
+ *
*/
public T get(String name) {
return get(name,0);
@@ -212,7 +216,10 @@ public class NamedList<T> implements Clo
/**
* Gets the value for the first instance of the specified name
* found starting at the specified index.
- *
+ * <p>
+ * NOTE: this runs in linear time (it scans starting at the
+ * specified position until it finds the first pair with
+ * the specified name).
* @return null if not found or if the value stored was null.
* @see #indexOf
*/
@@ -377,6 +384,11 @@ public class NamedList<T> implements Clo
return iter;
}
+ /**
+ * NOTE: this runs in linear time (it scans starting at the
+ * beginning of the list until it finds the first pair with
+ * the specified name).
+ */
public T remove(String name) {
int idx = indexOf(name, 0);
if(idx != -1) return remove(idx);
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/analysis/CollationKeyFilterFactory.java Mon Apr 4 13:03:42 2011
@@ -69,7 +69,9 @@ import org.apache.solr.util.plugin.Resou
* @see Locale
* @see RuleBasedCollator
* @since solr 3.1
+ * @deprecated use {@link org.apache.solr.schema.CollationField} instead.
*/
+@Deprecated
public class CollationKeyFilterFactory extends BaseTokenFilterFactory implements ResourceLoaderAware {
private Collator collator;
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/RequestHandlers.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/RequestHandlers.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/RequestHandlers.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/core/RequestHandlers.java Mon Apr 4 13:03:42 2011
@@ -71,18 +71,6 @@ final class RequestHandlers {
public RequestHandlers(SolrCore core) {
this.core = core;
- register(DEFAULT_HANDLER_NAME, getStandardHandler());
- }
-
- private SolrRequestHandler getStandardHandler(){
- SolrRequestHandler standard = core.createRequestHandler(SearchHandler.class.getName());
- NamedList defParams = new NamedList();
- defParams.add(CommonParams.HEADER_ECHO_PARAMS, EchoParamStyle.EXPLICIT.toString());
- defParams.add(CommonParams.ROWS, 10);
- NamedList nl = new NamedList();
- nl.add("defaults", defParams);
- standard.init(nl);
- return standard;
}
/**
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/request/UnInvertedField.java Mon Apr 4 13:03:42 2011
@@ -18,16 +18,11 @@
package org.apache.solr.request;
import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.DocTermOrds;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
-import org.apache.lucene.util.PagedBytes;
import org.apache.noggit.CharArr;
import org.apache.solr.common.params.FacetParams;
import org.apache.solr.common.util.NamedList;
@@ -44,15 +39,11 @@ import org.apache.solr.handler.component
import org.apache.solr.handler.component.FieldFacetStats;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.Bits;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.Map;
-import java.util.Comparator;
import java.util.concurrent.atomic.AtomicLong;
@@ -86,7 +77,7 @@ import java.util.concurrent.atomic.Atomi
* much like Lucene's own internal term index).
*
*/
-public class UnInvertedField {
+public class UnInvertedField extends DocTermOrds {
private static int TNUM_OFFSET=2;
static class TopTerm {
@@ -100,362 +91,109 @@ public class UnInvertedField {
}
}
- String field;
- int numTermsInField;
- int termsInverted; // number of unique terms that were un-inverted
- long termInstances; // total number of references to term numbers
- final TermIndex ti;
long memsz;
- int total_time; // total time to uninvert the field
- int phase1_time; // time for phase1 of the uninvert process
final AtomicLong use = new AtomicLong(); // number of uses
- int[] index;
- byte[][] tnums = new byte[256][];
- int[] maxTermCounts;
+ int[] maxTermCounts = new int[1024];
+
final Map<Integer,TopTerm> bigTerms = new LinkedHashMap<Integer,TopTerm>();
+ private SolrIndexSearcher.DocsEnumState deState;
+ private final SolrIndexSearcher searcher;
+
+ @Override
+ protected void visitTerm(TermsEnum te, int termNum) throws IOException {
+
+ if (termNum >= maxTermCounts.length) {
+ // resize by doubling - for very large number of unique terms, expanding
+ // by 4K and resultant GC will dominate uninvert times. Resize at end if material
+ int[] newMaxTermCounts = new int[maxTermCounts.length*2];
+ System.arraycopy(maxTermCounts, 0, newMaxTermCounts, 0, termNum);
+ maxTermCounts = newMaxTermCounts;
+ }
+
+ final BytesRef term = te.term();
+
+ if (te.docFreq() > maxTermDocFreq) {
+ TopTerm topTerm = new TopTerm();
+ topTerm.term = new BytesRef(term);
+ topTerm.termNum = termNum;
+ bigTerms.put(topTerm.termNum, topTerm);
+
+ if (deState == null) {
+ deState = new SolrIndexSearcher.DocsEnumState();
+ deState.termsEnum = te;
+ }
+
+ maxTermCounts[termNum] = searcher.getDocSet(new TermQuery(new Term(field, topTerm.term)), deState).size();
+ //System.out.println(" big term termNum=" + termNum + " term=" + topTerm.term.utf8ToString() + " size=" + maxTermCounts[termNum] + " dF=" + te.docFreq());
+ }
+ }
+
+ @Override
+ protected void setActualDocFreq(int termNum, int docFreq) {
+ maxTermCounts[termNum] = docFreq;
+ }
public long memSize() {
// can cache the mem size since it shouldn't change
if (memsz!=0) return memsz;
- long sz = 8*8 + 32; // local fields
+ long sz = super.ramUsedInBytes();
+ sz += 8*8 + 32; // local fields
sz += bigTerms.size() * 64;
for (TopTerm tt : bigTerms.values()) {
sz += tt.memSize();
}
- if (index != null) sz += index.length * 4;
- if (tnums!=null) {
- for (byte[] arr : tnums)
- if (arr != null) sz += arr.length;
- }
if (maxTermCounts != null)
sz += maxTermCounts.length * 4;
- sz += ti.memSize();
+ if (indexedTermsArray != null) {
+ // assume 8 byte references?
+ sz += 8+8+8+8+(indexedTermsArray.length<<3)+sizeOfIndexedStrings;
+ }
memsz = sz;
return sz;
}
-
- /** Number of bytes to represent an unsigned int as a vint. */
- static int vIntSize(int x) {
- if ((x & (0xffffffff << (7*1))) == 0 ) {
- return 1;
- }
- if ((x & (0xffffffff << (7*2))) == 0 ) {
- return 2;
- }
- if ((x & (0xffffffff << (7*3))) == 0 ) {
- return 3;
- }
- if ((x & (0xffffffff << (7*4))) == 0 ) {
- return 4;
- }
- return 5;
- }
-
-
- // todo: if we know the size of the vInt already, we could do
- // a single switch on the size
- static int writeInt(int x, byte[] arr, int pos) {
- int a;
- a = (x >>> (7*4));
- if (a != 0) {
- arr[pos++] = (byte)(a | 0x80);
- }
- a = (x >>> (7*3));
- if (a != 0) {
- arr[pos++] = (byte)(a | 0x80);
- }
- a = (x >>> (7*2));
- if (a != 0) {
- arr[pos++] = (byte)(a | 0x80);
- }
- a = (x >>> (7*1));
- if (a != 0) {
- arr[pos++] = (byte)(a | 0x80);
- }
- arr[pos++] = (byte)(x & 0x7f);
- return pos;
- }
-
-
-
public UnInvertedField(String field, SolrIndexSearcher searcher) throws IOException {
- this.field = field;
- this.ti = new TermIndex(field,
- TrieField.getMainValuePrefix(searcher.getSchema().getFieldType(field)));
- uninvert(searcher);
- }
-
-
- private void uninvert(SolrIndexSearcher searcher) throws IOException {
- long startTime = System.currentTimeMillis();
-
- IndexReader reader = searcher.getIndexReader();
- int maxDoc = reader.maxDoc();
-
- int[] index = new int[maxDoc]; // immediate term numbers, or the index into the byte[] representing the last number
- this.index = index;
- final int[] lastTerm = new int[maxDoc]; // last term we saw for this document
- final byte[][] bytes = new byte[maxDoc][]; // list of term numbers for the doc (delta encoded vInts)
- maxTermCounts = new int[1024];
-
- NumberedTermsEnum te = ti.getEnumerator(reader);
-
- // threshold, over which we use set intersections instead of counting
- // to (1) save memory, and (2) speed up faceting.
- // Add 2 for testing purposes so that there will always be some terms under
- // the threshold even when the index is very small.
- int threshold = maxDoc / 20 + 2;
- // threshold = 2000000000; //////////////////////////////// USE FOR TESTING
-
- // we need a minimum of 9 bytes, but round up to 12 since the space would
- // be wasted with most allocators anyway.
- byte[] tempArr = new byte[12];
-
- //
- // enumerate all terms, and build an intermediate form of the un-inverted field.
- //
- // During this intermediate form, every document has a (potential) byte[]
- // and the int[maxDoc()] array either contains the termNumber list directly
- // or the *end* offset of the termNumber list in it's byte array (for faster
- // appending and faster creation of the final form).
- //
- // idea... if things are too large while building, we could do a range of docs
- // at a time (but it would be a fair amount slower to build)
- // could also do ranges in parallel to take advantage of multiple CPUs
-
- // OPTIONAL: remap the largest df terms to the lowest 128 (single byte)
- // values. This requires going over the field first to find the most
- // frequent terms ahead of time.
-
- SolrIndexSearcher.DocsEnumState deState = null;
-
- for (;;) {
- BytesRef t = te.term();
- if (t==null) break;
-
- int termNum = te.getTermNumber();
-
- if (termNum >= maxTermCounts.length) {
- // resize by doubling - for very large number of unique terms, expanding
- // by 4K and resultant GC will dominate uninvert times. Resize at end if material
- int[] newMaxTermCounts = new int[maxTermCounts.length*2];
- System.arraycopy(maxTermCounts, 0, newMaxTermCounts, 0, termNum);
- maxTermCounts = newMaxTermCounts;
- }
-
- int df = te.docFreq();
- if (df >= threshold) {
- TopTerm topTerm = new TopTerm();
- topTerm.term = new BytesRef(t);
- topTerm.termNum = termNum;
- bigTerms.put(topTerm.termNum, topTerm);
-
- if (deState == null) {
- deState = new SolrIndexSearcher.DocsEnumState();
- deState.termsEnum = te.tenum;
- deState.reuse = te.docsEnum;
+ super(field,
+ // threshold, over which we use set intersections instead of counting
+ // to (1) save memory, and (2) speed up faceting.
+ // Add 1 for testing purposes so that there will always be some terms under
+ // the threshold even when the index is very
+ // small.
+ searcher.maxDoc()/20 + 1,
+ DEFAULT_INDEX_INTERVAL_BITS);
+ //System.out.println("maxTermDocFreq=" + maxTermDocFreq + " maxDoc=" + searcher.maxDoc());
+
+ final String prefix = TrieField.getMainValuePrefix(searcher.getSchema().getFieldType(field));
+ this.searcher = searcher;
+ try {
+ uninvert(searcher.getIndexReader(), prefix == null ? null : new BytesRef(prefix));
+ } catch (IllegalStateException ise) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ise.getMessage());
+ }
+ if (tnums != null) {
+ for(byte[] target : tnums) {
+ if (target != null && target.length > (1<<24)*.9) {
+ SolrCore.log.warn("Approaching too many values for UnInvertedField faceting on field '"+field+"' : bucket size=" + target.length);
}
- DocSet set = searcher.getDocSet(new TermQuery(new Term(ti.field, topTerm.term)), deState);
- te.docsEnum = deState.reuse;
-
- maxTermCounts[termNum] = set.size();
-
- te.next();
- continue;
}
-
- termsInverted++;
-
- DocsEnum docsEnum = te.getDocsEnum();
-
- DocsEnum.BulkReadResult bulkResult = docsEnum.getBulkResult();
-
- for(;;) {
- int n = docsEnum.read();
- if (n <= 0) break;
-
- maxTermCounts[termNum] += n;
-
- for (int i=0; i<n; i++) {
- termInstances++;
- int doc = bulkResult.docs.ints[i];
- // add 2 to the term number to make room for special reserved values:
- // 0 (end term) and 1 (index into byte array follows)
- int delta = termNum - lastTerm[doc] + TNUM_OFFSET;
- lastTerm[doc] = termNum;
- int val = index[doc];
-
- if ((val & 0xff)==1) {
- // index into byte array (actually the end of
- // the doc-specific byte[] when building)
- int pos = val >>> 8;
- int ilen = vIntSize(delta);
- byte[] arr = bytes[doc];
- int newend = pos+ilen;
- if (newend > arr.length) {
- // We avoid a doubling strategy to lower memory usage.
- // this faceting method isn't for docs with many terms.
- // In hotspot, objects have 2 words of overhead, then fields, rounded up to a 64-bit boundary.
- // TODO: figure out what array lengths we can round up to w/o actually using more memory
- // (how much space does a byte[] take up? Is data preceded by a 32 bit length only?
- // It should be safe to round up to the nearest 32 bits in any case.
- int newLen = (newend + 3) & 0xfffffffc; // 4 byte alignment
- byte[] newarr = new byte[newLen];
- System.arraycopy(arr, 0, newarr, 0, pos);
- arr = newarr;
- bytes[doc] = newarr;
- }
- pos = writeInt(delta, arr, pos);
- index[doc] = (pos<<8) | 1; // update pointer to end index in byte[]
- } else {
- // OK, this int has data in it... find the end (a zero starting byte - not
- // part of another number, hence not following a byte with the high bit set).
- int ipos;
- if (val==0) {
- ipos=0;
- } else if ((val & 0x0000ff80)==0) {
- ipos=1;
- } else if ((val & 0x00ff8000)==0) {
- ipos=2;
- } else if ((val & 0xff800000)==0) {
- ipos=3;
- } else {
- ipos=4;
- }
-
- int endPos = writeInt(delta, tempArr, ipos);
- if (endPos <= 4) {
- // value will fit in the integer... move bytes back
- for (int j=ipos; j<endPos; j++) {
- val |= (tempArr[j] & 0xff) << (j<<3);
- }
- index[doc] = val;
- } else {
- // value won't fit... move integer into byte[]
- for (int j=0; j<ipos; j++) {
- tempArr[j] = (byte)val;
- val >>>=8;
- }
- // point at the end index in the byte[]
- index[doc] = (endPos<<8) | 1;
- bytes[doc] = tempArr;
- tempArr = new byte[12];
- }
-
- }
-
- }
-
- }
-
- te.next();
}
- numTermsInField = te.getTermNumber();
- te.close();
-
// free space if outrageously wasteful (tradeoff memory/cpu)
-
if ((maxTermCounts.length - numTermsInField) > 1024) { // too much waste!
int[] newMaxTermCounts = new int[numTermsInField];
System.arraycopy(maxTermCounts, 0, newMaxTermCounts, 0, numTermsInField);
maxTermCounts = newMaxTermCounts;
- }
-
- long midPoint = System.currentTimeMillis();
-
- if (termInstances == 0) {
- // we didn't invert anything
- // lower memory consumption.
- index = this.index = null;
- tnums = null;
- } else {
-
- //
- // transform intermediate form into the final form, building a single byte[]
- // at a time, and releasing the intermediate byte[]s as we go to avoid
- // increasing the memory footprint.
- //
- for (int pass = 0; pass<256; pass++) {
- byte[] target = tnums[pass];
- int pos=0; // end in target;
- if (target != null) {
- pos = target.length;
- } else {
- target = new byte[4096];
- }
-
- // loop over documents, 0x00ppxxxx, 0x01ppxxxx, 0x02ppxxxx
- // where pp is the pass (which array we are building), and xx is all values.
- // each pass shares the same byte[] for termNumber lists.
- for (int docbase = pass<<16; docbase<maxDoc; docbase+=(1<<24)) {
- int lim = Math.min(docbase + (1<<16), maxDoc);
- for (int doc=docbase; doc<lim; doc++) {
- int val = index[doc];
- if ((val&0xff) == 1) {
- int len = val >>> 8;
- index[doc] = (pos<<8)|1; // change index to point to start of array
- if ((pos & 0xff000000) != 0) {
- // we only have 24 bits for the array index
- throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Too many values for UnInvertedField faceting on field "+field);
- }
- byte[] arr = bytes[doc];
- bytes[doc] = null; // IMPORTANT: allow GC to avoid OOM
- if (target.length <= pos + len) {
- int newlen = target.length;
- /*** we don't have to worry about the array getting too large
- * since the "pos" param will overflow first (only 24 bits available)
- if ((newlen<<1) <= 0) {
- // overflow...
- newlen = Integer.MAX_VALUE;
- if (newlen <= pos + len) {
- throw new SolrException(400,"Too many terms to uninvert field!");
- }
- } else {
- while (newlen <= pos + len) newlen<<=1; // doubling strategy
- }
- ****/
- while (newlen <= pos + len) newlen<<=1; // doubling strategy
- byte[] newtarget = new byte[newlen];
- System.arraycopy(target, 0, newtarget, 0, pos);
- target = newtarget;
- }
- System.arraycopy(arr, 0, target, pos, len);
- pos += len + 1; // skip single byte at end and leave it 0 for terminator
- }
- }
- }
-
- // shrink array
- if (pos < target.length) {
- byte[] newtarget = new byte[pos];
- System.arraycopy(target, 0, newtarget, 0, pos);
- target = newtarget;
- if (target.length > (1<<24)*.9) {
- SolrCore.log.warn("Approaching too many values for UnInvertedField faceting on field '"+field+"' : bucket size=" + target.length);
- }
- }
-
- tnums[pass] = target;
-
- if ((pass << 16) > maxDoc)
- break;
- }
}
- long endTime = System.currentTimeMillis();
-
- total_time = (int)(endTime-startTime);
- phase1_time = (int)(midPoint-startTime);
-
SolrCore.log.info("UnInverted multi-valued field " + toString());
+ //System.out.println("CREATED: " + toString() + " ti.index=" + ti.index);
}
-
-
+ public int getNumTerms() {
+ return numTermsInField;
+ }
public NamedList<Integer> getCounts(SolrIndexSearcher searcher, DocSet baseDocs, int offset, int limit, Integer mincount, boolean missing, String sort, String prefix) throws IOException {
use.incrementAndGet();
@@ -468,6 +206,7 @@ public class UnInvertedField {
int baseSize = docs.size();
int maxDoc = searcher.maxDoc();
+ //System.out.println("GET COUNTS field=" + field + " baseSize=" + baseSize + " minCount=" + mincount + " maxDoc=" + maxDoc + " numTermsInField=" + numTermsInField);
if (baseSize >= mincount) {
final int[] index = this.index;
@@ -481,14 +220,20 @@ public class UnInvertedField {
int startTerm = 0;
int endTerm = numTermsInField; // one past the end
- NumberedTermsEnum te = ti.getEnumerator(searcher.getIndexReader());
+ TermsEnum te = getOrdTermsEnum(searcher.getIndexReader());
if (prefix != null && prefix.length() > 0) {
BytesRef prefixBr = new BytesRef(prefix);
- te.skipTo(prefixBr);
- startTerm = te.getTermNumber();
+ if (te.seek(prefixBr, true) == TermsEnum.SeekStatus.END) {
+ startTerm = numTermsInField;
+ } else {
+ startTerm = (int) te.ord();
+ }
prefixBr.append(ByteUtils.bigTerm);
- te.skipTo(prefixBr);
- endTerm = te.getTermNumber();
+ if (te.seek(prefixBr, true) == TermsEnum.SeekStatus.END) {
+ endTerm = numTermsInField;
+ } else {
+ endTerm = (int) te.ord();
+ }
}
/***********
@@ -514,13 +259,18 @@ public class UnInvertedField {
docs = new BitDocSet(bs, maxDoc - baseSize);
// simply negating will mean that we have deleted docs in the set.
// that should be OK, as their entries in our table should be empty.
+ //System.out.println(" NEG");
}
// For the biggest terms, do straight set intersections
for (TopTerm tt : bigTerms.values()) {
+ //System.out.println(" do big termNum=" + tt.termNum + " term=" + tt.term.utf8ToString());
// TODO: counts could be deferred if sorted==false
if (tt.termNum >= startTerm && tt.termNum < endTerm) {
- counts[tt.termNum] = searcher.numDocs(new TermQuery(new Term(ti.field, tt.term)), docs);
+ counts[tt.termNum] = searcher.numDocs(new TermQuery(new Term(field, tt.term)), docs);
+ //System.out.println(" count=" + counts[tt.termNum]);
+ } else {
+ //System.out.println("SKIP term=" + tt.termNum);
}
}
@@ -537,9 +287,11 @@ public class UnInvertedField {
DocIterator iter = docs.iterator();
while (iter.hasNext()) {
int doc = iter.nextDoc();
+ //System.out.println("iter doc=" + doc);
int code = index[doc];
if ((code & 0xff)==1) {
+ //System.out.println(" ptr");
int pos = code>>>8;
int whichArray = (doc >>> 16) & 0xff;
byte[] arr = tnums[whichArray];
@@ -553,9 +305,11 @@ public class UnInvertedField {
}
if (delta == 0) break;
tnum += delta - TNUM_OFFSET;
+ //System.out.println(" tnum=" + tnum);
counts[tnum]++;
}
} else {
+ //System.out.println(" inlined");
int tnum = 0;
int delta = 0;
for (;;) {
@@ -563,6 +317,7 @@ public class UnInvertedField {
if ((code & 0x80)==0) {
if (delta==0) break;
tnum += delta - TNUM_OFFSET;
+ //System.out.println(" tnum=" + tnum);
counts[tnum]++;
delta = 0;
}
@@ -583,6 +338,7 @@ public class UnInvertedField {
LongPriorityQueue queue = new LongPriorityQueue(Math.min(maxsize,1000), maxsize, Long.MIN_VALUE);
int min=mincount-1; // the smallest value in the top 'N' values
+ //System.out.println("START=" + startTerm + " END=" + endTerm);
for (int i=startTerm; i<endTerm; i++) {
int c = doNegative ? maxTermCounts[i] - counts[i] : counts[i];
if (c>min) {
@@ -641,11 +397,14 @@ public class UnInvertedField {
}
});
- // convert the term numbers to term values and set as the label
+ // convert the term numbers to term values and set
+ // as the label
+ //System.out.println("sortStart=" + sortedIdxStart + " end=" + sortedIdxEnd);
for (int i=sortedIdxStart; i<sortedIdxEnd; i++) {
int idx = indirect[i];
int tnum = (int)sorted[idx];
String label = getReadableValue(getTermValue(te, tnum), ft, spare);
+ //System.out.println(" label=" + label);
res.setName(idx - sortedIdxStart, label);
}
@@ -668,8 +427,6 @@ public class UnInvertedField {
res.add(label, c);
}
}
-
- te.close();
}
@@ -678,6 +435,8 @@ public class UnInvertedField {
res.add(null, SimpleFacets.getFieldMissingCount(searcher, baseDocs, field));
}
+ //System.out.println(" res=" + res);
+
return res;
}
@@ -731,8 +490,7 @@ public class UnInvertedField {
final int[] index = this.index;
final int[] counts = new int[numTermsInField];//keep track of the number of times we see each word in the field for all the documents in the docset
- NumberedTermsEnum te = ti.getEnumerator(searcher.getIndexReader());
-
+ TermsEnum te = getOrdTermsEnum(searcher.getIndexReader());
boolean doNegative = false;
if (finfo.length == 0) {
@@ -755,7 +513,7 @@ public class UnInvertedField {
for (TopTerm tt : bigTerms.values()) {
// TODO: counts could be deferred if sorted==false
if (tt.termNum >= 0 && tt.termNum < numTermsInField) {
- final Term t = new Term(ti.field, tt.term);
+ final Term t = new Term(field, tt.term);
if (finfo.length == 0) {
counts[tt.termNum] = searcher.numDocs(new TermQuery(t), docs);
} else {
@@ -836,7 +594,6 @@ public class UnInvertedField {
f.accumulateTermNum(i, value);
}
}
- te.close();
int c = missing.size();
allstats.addMissing(c);
@@ -870,23 +627,26 @@ public class UnInvertedField {
}
/** may return a reused BytesRef */
- BytesRef getTermValue(NumberedTermsEnum te, int termNum) throws IOException {
+ BytesRef getTermValue(TermsEnum te, int termNum) throws IOException {
+ //System.out.println("getTermValue termNum=" + termNum + " this=" + this + " numTerms=" + numTermsInField);
if (bigTerms.size() > 0) {
// see if the term is one of our big terms.
TopTerm tt = bigTerms.get(termNum);
if (tt != null) {
+ //System.out.println(" return big " + tt.term);
return tt.term;
}
}
- return te.skipTo(termNum);
+ return lookupTerm(te, termNum);
}
@Override
public String toString() {
+ final long indexSize = indexedTermsArray == null ? 0 : (8+8+8+8+(indexedTermsArray.length<<3)+sizeOfIndexedStrings); // assume 8 byte references?
return "{field=" + field
+ ",memSize="+memSize()
- + ",tindexSize="+ti.memSize()
+ + ",tindexSize="+indexSize
+ ",time="+total_time
+ ",phase1="+phase1_time
+ ",nTerms="+numTermsInField
@@ -896,7 +656,6 @@ public class UnInvertedField {
+ "}";
}
-
//////////////////////////////////////////////////////////////////
//////////////////////////// caching /////////////////////////////
//////////////////////////////////////////////////////////////////
@@ -920,287 +679,3 @@ public class UnInvertedField {
return uif;
}
}
-
-
-// How to share TermDocs (int[] score[])???
-// Hot to share TermPositions?
-/***
-class TermEnumListener {
- void doTerm(Term t) {
- }
- void done() {
- }
-}
-***/
-
-
-class NumberedTermsEnum extends TermsEnum {
- protected final IndexReader reader;
- protected final TermIndex tindex;
- protected TermsEnum tenum;
- protected int pos=-1;
- protected BytesRef termText;
- protected DocsEnum docsEnum;
- protected Bits deletedDocs;
-
-
- NumberedTermsEnum(IndexReader reader, TermIndex tindex) throws IOException {
- this.reader = reader;
- this.tindex = tindex;
- }
-
-
- NumberedTermsEnum(IndexReader reader, TermIndex tindex, BytesRef termValue, int pos) throws IOException {
- this.reader = reader;
- this.tindex = tindex;
- this.pos = pos;
- Terms terms = MultiFields.getTerms(reader, tindex.field);
- deletedDocs = MultiFields.getDeletedDocs(reader);
- if (terms != null) {
- tenum = terms.iterator();
- tenum.seek(termValue);
- setTerm();
- }
- }
-
- @Override
- public Comparator<BytesRef> getComparator() throws IOException {
- return tenum.getComparator();
- }
-
- public DocsEnum getDocsEnum() throws IOException {
- docsEnum = tenum.docs(deletedDocs, docsEnum);
- return docsEnum;
- }
-
- protected BytesRef setTerm() throws IOException {
- termText = tenum.term();
- if (tindex.prefix != null && !termText.startsWith(tindex.prefix)) {
- termText = null;
- }
- return termText;
- }
-
- @Override
- public BytesRef next() throws IOException {
- pos++;
- if (tenum.next() == null) {
- termText = null;
- return null;
- }
- return setTerm(); // this is extra work if we know we are in bounds...
- }
-
- @Override
- public BytesRef term() {
- return termText;
- }
-
- @Override
- public int docFreq() throws IOException {
- return tenum.docFreq();
- }
-
- @Override
- public long totalTermFreq() throws IOException {
- return tenum.totalTermFreq();
- }
-
- public BytesRef skipTo(BytesRef target) throws IOException {
-
- // already here
- if (termText != null && termText.equals(target)) return termText;
-
- if (tenum == null) {
- return null;
- }
-
- int startIdx = Arrays.binarySearch(tindex.index,target);
-
- if (startIdx >= 0) {
- // we hit the term exactly... lucky us!
- TermsEnum.SeekStatus seekStatus = tenum.seek(target);
- assert seekStatus == TermsEnum.SeekStatus.FOUND;
- pos = startIdx << tindex.intervalBits;
- return setTerm();
- }
-
- // we didn't hit the term exactly
- startIdx=-startIdx-1;
-
- if (startIdx == 0) {
- // our target occurs *before* the first term
- TermsEnum.SeekStatus seekStatus = tenum.seek(target);
- assert seekStatus == TermsEnum.SeekStatus.NOT_FOUND;
- pos = 0;
- return setTerm();
- }
-
- // back up to the start of the block
- startIdx--;
-
- if ((pos >> tindex.intervalBits) == startIdx && termText != null && termText.compareTo(target)<=0) {
- // we are already in the right block and the current term is before the term we want,
- // so we don't need to seek.
- } else {
- // seek to the right block
- TermsEnum.SeekStatus seekStatus = tenum.seek(tindex.index[startIdx]);
- assert seekStatus == TermsEnum.SeekStatus.FOUND;
- pos = startIdx << tindex.intervalBits;
- setTerm(); // should be non-null since it's in the index
- }
-
- while (termText != null && termText.compareTo(target) < 0) {
- next();
- }
-
- return termText;
- }
-
- public BytesRef skipTo(int termNumber) throws IOException {
- int delta = termNumber - pos;
- if (delta < 0 || delta > tindex.interval || tenum==null) {
- int idx = termNumber >>> tindex.intervalBits;
- BytesRef base = tindex.index[idx];
- pos = idx << tindex.intervalBits;
- delta = termNumber - pos;
- TermsEnum.SeekStatus seekStatus = tenum.seek(base);
- assert seekStatus == TermsEnum.SeekStatus.FOUND;
- }
- while (--delta >= 0) {
- BytesRef br = tenum.next();
- if (br == null) {
- termText = null;
- return null;
- }
- ++pos;
- }
- return setTerm();
- }
-
- protected void close() throws IOException {
- // no-op, needed so the anon subclass that does indexing
- // can build its index
- }
-
- /** The current term number, starting at 0.
- * Only valid if the previous call to next() or skipTo() returned true.
- */
- public int getTermNumber() {
- return pos;
- }
-
- @Override
- public long ord() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SeekStatus seek(long ord) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DocsEnum docs(Bits skipDocs, DocsEnum reuse) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public SeekStatus seek(BytesRef target, boolean useCache) {
- throw new UnsupportedOperationException();
- }
-}
-
-
-/**
- * Class to save memory by only storing every nth term (for random access), while
- * numbering the terms, allowing them to be retrieved later by number.
- * This is only valid when used with the IndexReader it was created with.
- * The IndexReader is not actually stored to facilitate caching by using it as a key in
- * a weak hash map.
- */
-class TermIndex {
- final static int intervalBits = 7; // decrease to a low number like 2 for testing
- final static int intervalMask = 0xffffffff >>> (32-intervalBits);
- final static int interval = 1 << intervalBits;
-
- final String field;
- final BytesRef prefix;
- BytesRef[] index;
- int nTerms;
- long sizeOfStrings;
-
- TermIndex(String field) {
- this(field, null);
- }
-
- TermIndex(String field, String prefix) {
- this.field = field;
- this.prefix = prefix == null ? null : new BytesRef(prefix);
- }
-
- NumberedTermsEnum getEnumerator(IndexReader reader, int termNumber) throws IOException {
- NumberedTermsEnum te = new NumberedTermsEnum(reader, this);
- te.skipTo(termNumber);
- return te;
- }
-
- /* The first time an enumerator is requested, it should be used
- with next() to fully traverse all of the terms so the index
- will be built.
- */
- NumberedTermsEnum getEnumerator(IndexReader reader) throws IOException {
- if (index==null) return new NumberedTermsEnum(reader,this, prefix==null?new BytesRef():prefix, 0) {
- ArrayList<BytesRef> lst;
- PagedBytes bytes;
-
- @Override
- protected BytesRef setTerm() throws IOException {
- BytesRef br = super.setTerm();
- if (br != null && (pos & intervalMask)==0) {
- sizeOfStrings += br.length;
- if (lst==null) {
- lst = new ArrayList<BytesRef>();
- bytes = new PagedBytes(15);
- }
- BytesRef out = new BytesRef();
- bytes.copy(br, out);
- lst.add(out);
- }
- return br;
- }
-
- @Override
- public BytesRef skipTo(int termNumber) throws IOException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() throws IOException {
- nTerms=pos;
- super.close();
- index = lst!=null ? lst.toArray(new BytesRef[lst.size()]) : new BytesRef[0];
- }
- };
- else return new NumberedTermsEnum(reader,this,new BytesRef(),0);
- }
-
-
- /**
- * Returns the approximate amount of memory taken by this TermIndex.
- * This is only an approximation and doesn't take into account java object overhead.
- *
- * @return
- * the approximate memory consumption in bytes
- */
- public long memSize() {
- // assume 8 byte references?
- return 8+8+8+8+(index.length<<3)+sizeOfStrings;
- }
-}
-
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/DocIdAugmenterFactory.java Mon Apr 4 13:03:42 2011
@@ -19,6 +19,7 @@ package org.apache.solr.response.transfo
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.request.SolrQueryRequest;
/**
* @version $Id$
@@ -27,7 +28,7 @@ import org.apache.solr.common.SolrExcept
public class DocIdAugmenterFactory extends TransformerFactory
{
@Override
- public DocTransformer create(String field, String arg) {
+ public DocTransformer create(String field, String arg, SolrQueryRequest req) {
if( arg != null ) {
throw new SolrException( ErrorCode.BAD_REQUEST,
"DocIdAugmenter does not take any arguments" );
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ExplainAugmenterFactory.java Mon Apr 4 13:03:42 2011
@@ -23,6 +23,7 @@ import org.apache.solr.common.SolrDocume
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.SolrPluginUtils;
/**
@@ -62,7 +63,7 @@ public class ExplainAugmenterFactory ext
}
@Override
- public DocTransformer create(String field, String arg) {
+ public DocTransformer create(String field, String arg, SolrQueryRequest req) {
Style style = (arg==null)?defaultStyle:getStyle(arg);
return new ExplainAugmenter( field, style );
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ShardAugmenterFactory.java Mon Apr 4 13:03:42 2011
@@ -16,6 +16,8 @@
*/
package org.apache.solr.response.transform;
+import org.apache.solr.request.SolrQueryRequest;
+
/**
* @version $Id$
@@ -24,7 +26,7 @@ package org.apache.solr.response.transfo
public class ShardAugmenterFactory extends TransformerFactory
{
@Override
- public DocTransformer create(String field, String arg) {
+ public DocTransformer create(String field, String arg, SolrQueryRequest req) {
String id = "TODO... find ID";
// Maybe it is stored in the context?
// is it a request variable?
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/TransformerFactory.java Mon Apr 4 13:03:42 2011
@@ -20,7 +20,9 @@ package org.apache.solr.response.transfo
import java.util.HashMap;
import java.util.Map;
+import org.apache.solr.common.params.SolrParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
import org.apache.solr.util.plugin.NamedListInitializedPlugin;
/**
@@ -36,7 +38,7 @@ public abstract class TransformerFactory
defaultUserArgs = (String)args.get( "args" );
}
- public abstract DocTransformer create(String field, String args);
+ public abstract DocTransformer create(String field, String args, SolrQueryRequest req);
public static final Map<String,TransformerFactory> defaultFactories = new HashMap<String,TransformerFactory>();
static {
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueAugmenterFactory.java Mon Apr 4 13:03:42 2011
@@ -21,6 +21,7 @@ import org.apache.solr.common.SolrExcept
import org.apache.solr.common.SolrException.ErrorCode;
import org.apache.solr.common.util.DateUtil;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.request.SolrQueryRequest;
/**
* @version $Id$
@@ -60,7 +61,7 @@ public class ValueAugmenterFactory exten
}
@Override
- public DocTransformer create(String field, String arg) {
+ public DocTransformer create(String field, String arg, SolrQueryRequest req) {
Object val = value;
if( val == null ) {
val = (arg==null)?defaultValue:getObjectFrom(arg);
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/response/transform/ValueSourceAugmenter.java Mon Apr 4 13:03:42 2011
@@ -16,10 +16,19 @@
*/
package org.apache.solr.response.transform;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.ReaderUtil;
import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.core.SolrCore;
import org.apache.solr.search.QParser;
+import org.apache.solr.search.SolrIndexSearcher;
+import org.apache.solr.search.function.DocValues;
import org.apache.solr.search.function.ValueSource;
+import java.io.IOException;
+import java.util.Map;
+
/**
* Add values from a ValueSource (function query etc)
*
@@ -32,13 +41,15 @@ public class ValueSourceAugmenter extend
{
public final String name;
public final QParser qparser;
- public final ValueSource values;
+ public final ValueSource valueSource;
+
+
- public ValueSourceAugmenter( String name, QParser qparser, ValueSource values )
+ public ValueSourceAugmenter( String name, QParser qparser, ValueSource valueSource )
{
this.name = name;
this.qparser = qparser;
- this.values = values;
+ this.valueSource = valueSource;
}
@Override
@@ -49,14 +60,41 @@ public class ValueSourceAugmenter extend
@Override
public void setContext( TransformContext context ) {
- // maybe we do something here?
+ IndexReader reader = qparser.getReq().getSearcher().getIndexReader();
+ readerContexts = reader.getTopReaderContext().leaves();
+ docValuesArr = new DocValues[readerContexts.length];
+
+ searcher = qparser.getReq().getSearcher();
+ this.fcontext = valueSource.newContext(searcher);
}
+
+ Map fcontext;
+ SolrIndexSearcher searcher;
+ IndexReader.AtomicReaderContext[] readerContexts;
+ DocValues docValuesArr[];
+
+
@Override
public void transform(SolrDocument doc, int docid) {
- // TODO, should know what the real type is -- not always string
- // how do we get to docvalues?
- Object v = "now what..."; //values.g.strVal( docid );
- doc.setField( name, v );
+ // This is only good for random-access functions
+
+ try {
+
+ // TODO: calculate this stuff just once across diff functions
+ int idx = ReaderUtil.subIndex(docid, readerContexts);
+ IndexReader.AtomicReaderContext rcontext = readerContexts[idx];
+ DocValues values = docValuesArr[idx];
+ if (values == null) {
+ docValuesArr[idx] = values = valueSource.getValues(fcontext, rcontext);
+ }
+
+ int localId = docid - rcontext.docBase;
+ float val = values.floatVal(localId); // TODO: handle all types -- see: SOLR-2443
+
+ doc.setField( name, val );
+ } catch (IOException e) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "exception at docid " + docid + " for valuesource " + valueSource, e, false);
+ }
}
}
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/DateField.java Mon Apr 4 13:03:42 2011
@@ -180,6 +180,14 @@ public class DateField extends FieldType
"Invalid Date Math String:'" +val+'\'',e);
}
}
+
+ public Fieldable createField(SchemaField field, Object value, float boost) {
+ // Convert to a string before indexing
+ if(value instanceof Date) {
+ value = toInternal( (Date)value ) + 'Z';
+ }
+ return super.createField(field, value, boost);
+ }
public String toInternal(Date val) {
return formatDate(val);
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldProperties.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldProperties.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldProperties.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldProperties.java Mon Apr 4 13:03:42 2011
@@ -22,29 +22,31 @@ import java.util.HashMap;
/**
* @version $Id$
+ *
+ * @lucene.internal
*/
-abstract class FieldProperties {
+public abstract class FieldProperties {
// use a bitfield instead of many different boolean variables since
// many of the variables are independent or semi-independent.
// bit values for boolean field properties.
- final static int INDEXED = 0x00000001;
- final static int TOKENIZED = 0x00000002;
- final static int STORED = 0x00000004;
- final static int BINARY = 0x00000008;
- final static int OMIT_NORMS = 0x00000010;
- final static int OMIT_TF_POSITIONS = 0x00000020;
- final static int STORE_TERMVECTORS = 0x00000040;
- final static int STORE_TERMPOSITIONS = 0x00000080;
- final static int STORE_TERMOFFSETS = 0x00000100;
+ protected final static int INDEXED = 0x00000001;
+ protected final static int TOKENIZED = 0x00000002;
+ protected final static int STORED = 0x00000004;
+ protected final static int BINARY = 0x00000008;
+ protected final static int OMIT_NORMS = 0x00000010;
+ protected final static int OMIT_TF_POSITIONS = 0x00000020;
+ protected final static int STORE_TERMVECTORS = 0x00000040;
+ protected final static int STORE_TERMPOSITIONS = 0x00000080;
+ protected final static int STORE_TERMOFFSETS = 0x00000100;
- final static int MULTIVALUED = 0x00000200;
- final static int SORT_MISSING_FIRST = 0x00000400;
- final static int SORT_MISSING_LAST = 0x00000800;
+ protected final static int MULTIVALUED = 0x00000200;
+ protected final static int SORT_MISSING_FIRST = 0x00000400;
+ protected final static int SORT_MISSING_LAST = 0x00000800;
- final static int REQUIRED = 0x00001000;
+ protected final static int REQUIRED = 0x00001000;
static final String[] propertyNames = {
"indexed", "tokenized", "stored",
Modified: lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java?rev=1088606&r1=1088605&r2=1088606&view=diff
==============================================================================
--- lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java (original)
+++ lucene/dev/branches/realtime_search/solr/src/java/org/apache/solr/schema/FieldType.java Mon Apr 4 13:03:42 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.Similarity;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TermRangeQuery;
@@ -83,6 +84,11 @@ public abstract class FieldType extends
public boolean isMultiValued() {
return (properties & MULTIVALUED) != 0;
}
+
+ /** Check if a property is set */
+ protected boolean hasProperty( int p ) {
+ return (properties & p) != 0;
+ }
/**
* A "polyField" is a FieldType that can produce more than one Fieldable instance for a single value, via the {@link #createFields(org.apache.solr.schema.SchemaField, Object, float)} method. This is useful
@@ -503,6 +509,34 @@ public abstract class FieldType extends
throw e;
}
+ /** @lucene.internal */
+ protected Similarity similarity;
+
+ /**
+ * Gets the Similarity used when scoring fields of this type
+ *
+ * <p>
+ * The default implementation returns null, which means this type
+ * has no custom similarity associated with it.
+ * </p>
+ *
+ * This method exists to internally support SolrSimilarityProvider.
+ * Custom application code interested in a field's Similarity should
+ * instead query via the searcher's SimilarityProvider.
+ * @lucene.internal
+ */
+ public Similarity getSimilarity() {
+ return similarity;
+ }
+
+ /**
+ * Sets the Similarity used when scoring fields of this type
+ * @lucene.internal
+ */
+ public void setSimilarity(Similarity similarity) {
+ this.similarity = similarity;
+ }
+
/**
* calls back to TextResponseWriter to write the field value
*/