You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2012/01/07 19:23:14 UTC
svn commit: r1228693 [3/5] - in /lucene/dev/branches/solrcloud: ./
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/benchmark/ dev-tools/maven/modules/benchmark/
lucene/ lucene/contrib/ lucene/contrib/highlighter/sr...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/IOUtils.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/IOUtils.java Sat Jan 7 18:23:08 2012
@@ -30,6 +30,8 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
+import org.apache.lucene.store.Directory;
+
/** This class emulates the new Java 7 "Try-With-Resources" statement.
* Remove once Lucene is on Java 7.
* @lucene.internal */
@@ -318,6 +320,16 @@ public final class IOUtils {
}
}
}
+
+ public static void deleteFilesIgnoringExceptions(Directory dir, String... files) {
+ for (String name : files) {
+ try {
+ dir.deleteFile(name);
+ } catch (IOException ignored) {
+ // ignore
+ }
+ }
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/automaton/UTF32ToUTF8.java Sat Jan 7 18:23:08 2012
@@ -310,17 +310,4 @@ public final class UTF32ToUTF8 {
utf8StateCount++;
return s;
}
-
- public static void main(String[] args) {
- final int startCode = Integer.parseInt(args[0]);
- final int endCode = Integer.parseInt(args[1]);
-
- Automaton a = new Automaton();
- State start = a.getInitialState();
- State end = new State();
- end.setAccept(true);
-
- UTF32ToUTF8 converter = new UTF32ToUTF8();
- converter.convertOneEdge(start, end, startCode, endCode);
- }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/contributions.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/contributions.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/contributions.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/contributions.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -727,5 +730,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -342,5 +345,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo2.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo2.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo2.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/demo2.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -392,5 +395,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/fileformats.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/fileformats.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/fileformats.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/fileformats.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -2623,5 +2626,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/gettingstarted.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/gettingstarted.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/gettingstarted.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/gettingstarted.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -281,5 +284,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/index.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/index.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/index.html Sat Jan 7 18:23:08 2012
@@ -197,6 +197,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit">
<hr>
@@ -251,5 +254,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/linkmap.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/linkmap.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/linkmap.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/linkmap.html Sat Jan 7 18:23:08 2012
@@ -197,6 +197,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -421,6 +424,12 @@ document.write("Last Published: " + docu
<a href="http://wiki.apache.org/lucene-java">Wiki</a> ___________________ <em>wiki</em>
</li>
</ul>
+
+<ul>
+<li>
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a> ___________________ <em>privacy</em>
+</li>
+</ul>
</ul>
</ul>
@@ -455,5 +464,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/lucene-contrib/index.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/lucene-contrib/index.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/lucene-contrib/index.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/lucene-contrib/index.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit">
<hr>
@@ -400,5 +403,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/queryparsersyntax.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/queryparsersyntax.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/queryparsersyntax.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/queryparsersyntax.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -507,5 +510,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/scoring.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/scoring.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/scoring.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/scoring.html Sat Jan 7 18:23:08 2012
@@ -199,6 +199,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -570,5 +573,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/build/site/systemrequirements.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/build/site/systemrequirements.html?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/build/site/systemrequirements.html (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/build/site/systemrequirements.html Sat Jan 7 18:23:08 2012
@@ -197,6 +197,9 @@ document.write("Last Published: " + docu
<div class="menuitem">
<a href="http://wiki.apache.org/lucene-java">Wiki</a>
</div>
+<div class="menuitem">
+<a href="http://lucene.apache.org/privacy.html">Privacy Policy</a>
+</div>
</div>
<div id="credit"></div>
<div id="roundbottom">
@@ -270,5 +273,18 @@ document.write("Last Published: " + docu
|end bottomstrip
+-->
</div>
+<script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/content/xdocs/site.xml?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/content/xdocs/site.xml (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/content/xdocs/site.xml Sat Jan 7 18:23:08 2012
@@ -78,6 +78,7 @@ See http://forrest.apache.org/docs/linki
<query-syntax label="Query Syntax" href="queryparsersyntax.html"/>
<scoring label="Scoring" href="scoring.html"/>
<wiki label="Wiki" href="ext:wiki" />
+ <privacy label="Privacy Policy" href="ext:privacy"/>
</docs>
<!--
@@ -113,7 +114,7 @@ See http://forrest.apache.org/docs/linki
<webapp href="docs/your-project.html#webapp"/>
<dtd-docs href="docs/dtd-docs.html"/>
</forrest>
-
+ <privacy href="http://lucene.apache.org/privacy.html"/>
<cocoon href="http://cocoon.apache.org/"/>
<xml.apache.org href="http://xml.apache.org/"/>
<issues href="http://issues.apache.org/jira/browse/LUCENE"/>
Modified: lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/skins/lucene/xslt/html/site-to-xhtml.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/skins/lucene/xslt/html/site-to-xhtml.xsl?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/skins/lucene/xslt/html/site-to-xhtml.xsl (original)
+++ lucene/dev/branches/solrcloud/lucene/src/site/src/documentation/skins/lucene/xslt/html/site-to-xhtml.xsl Sat Jan 7 18:23:08 2012
@@ -374,6 +374,19 @@ footer, searchbar, css etc. As input, i
|end bottomstrip
+</xsl:comment>
</div>
+ <script type="text/javascript">
+
+ var _gaq = _gaq || [];
+ _gaq.push(['_setAccount', 'UA-94576-12']);
+ _gaq.push(['_trackPageview']);
+
+ (function() {
+ var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
+ ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
+ var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
+ })();
+
+</script>
</body>
</html>
</xsl:template>
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java Sat Jan 7 18:23:08 2012
@@ -249,7 +249,46 @@ public abstract class BaseTokenStreamTes
// TODO: add a MockCharStream, and use it here too, to ensure that correctOffset etc is being done by tokenizers.
public static void checkRandomData(Random random, Analyzer a, int iterations) throws IOException {
checkRandomData(random, a, iterations, 20);
+ // now test with multiple threads
+ int numThreads = _TestUtil.nextInt(random, 4, 8);
+ Thread threads[] = new Thread[numThreads];
+ for (int i = 0; i < threads.length; i++) {
+ threads[i] = new AnalysisThread(new Random(random.nextLong()), a, iterations);
+ }
+ for (int i = 0; i < threads.length; i++) {
+ threads[i].start();
+ }
+ for (int i = 0; i < threads.length; i++) {
+ try {
+ threads[i].join();
+ } catch (InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+ }
}
+
+ static class AnalysisThread extends Thread {
+ final int iterations;
+ final Random random;
+ final Analyzer a;
+
+ AnalysisThread(Random random, Analyzer a, int iterations) {
+ this.random = random;
+ this.a = a;
+ this.iterations = iterations;
+ }
+
+ @Override
+ public void run() {
+ try {
+ // see the part in checkRandomData where it replays the same text again
+ // to verify reproducability/reuse: hopefully this would catch thread hazards.
+ checkRandomData(random, a, iterations, 20);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength) throws IOException {
for (int i = 0; i < iterations; i++) {
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/codecs/preflexrw/PreFlexRWCodec.java Sat Jan 7 18:23:08 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.codecs.preflex
* limitations under the License.
*/
+import org.apache.lucene.codecs.NormsFormat;
import org.apache.lucene.codecs.PostingsFormat;
import org.apache.lucene.codecs.lucene3x.Lucene3xCodec;
import org.apache.lucene.util.LuceneTestCase;
@@ -27,7 +28,8 @@ import org.apache.lucene.util.LuceneTest
*/
public class PreFlexRWCodec extends Lucene3xCodec {
private final PostingsFormat postings = new PreFlexRWPostingsFormat();
-
+ private final NormsFormat norms = new PreFlexRWNormsFormat();
+
@Override
public PostingsFormat postingsFormat() {
if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
@@ -36,4 +38,13 @@ public class PreFlexRWCodec extends Luce
return super.postingsFormat();
}
}
+
+ @Override
+ public NormsFormat normsFormat() {
+ if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) {
+ return norms;
+ } else {
+ return super.normsFormat();
+ }
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearch.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearch.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearch.java Sat Jan 7 18:23:08 2012
@@ -24,8 +24,6 @@ import java.io.PrintWriter;
import java.io.StringWriter;
import org.apache.lucene.util.LuceneTestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.lucene.store.*;
import org.apache.lucene.document.*;
@@ -36,11 +34,6 @@ import org.apache.lucene.search.*;
/** JUnit adaptation of an older test case SearchTest. */
public class TestSearch extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run (new TestSuite(TestSearch.class));
- }
-
/** This test performs a number of searches. It also compares output
* of searches using multi-file index segments with single-file
* index segments.
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/TestSearchForDuplicates.java Sat Jan 7 18:23:08 2012
@@ -28,18 +28,9 @@ import org.apache.lucene.analysis.*;
import org.apache.lucene.index.*;
import org.apache.lucene.search.*;
import org.apache.lucene.util.LuceneTestCase;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
public class TestSearchForDuplicates extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run (new TestSuite(TestSearchForDuplicates.class));
- }
-
-
-
static final String PRIORITY_FIELD ="priority";
static final String ID_FIELD ="id";
static final String HIGH_PRIORITY ="high";
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/codecs/lucene40/TestDocValues.java Sat Jan 7 18:23:08 2012
@@ -20,10 +20,11 @@ package org.apache.lucene.codecs.lucene4
import java.io.IOException;
import java.util.Comparator;
+import org.apache.lucene.codecs.DocValuesConsumer;
import org.apache.lucene.codecs.lucene40.values.Bytes;
import org.apache.lucene.codecs.lucene40.values.Floats;
import org.apache.lucene.codecs.lucene40.values.Ints;
-import org.apache.lucene.codecs.lucene40.values.Writer;
+import org.apache.lucene.index.DocValue;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.DocValues.Source;
@@ -58,12 +59,13 @@ public class TestDocValues extends Lucen
public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize)
throws IOException {
-
+ DocValueHolder valueHolder = new DocValueHolder();
+ valueHolder.comp = COMP;
final BytesRef bytesRef = new BytesRef();
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random),
+ DocValuesConsumer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random),
random.nextBoolean());
int maxDoc = 220;
final String[] values = new String[maxDoc];
@@ -79,7 +81,8 @@ public class TestDocValues extends Lucen
values[2 * i] = s;
UnicodeUtil.UTF16toUTF8(s, 0, s.length(), bytesRef);
- w.add(2 * i, bytesRef);
+ valueHolder.bytes = bytesRef;
+ w.add(2 * i, valueHolder);
}
w.finish(maxDoc);
assertEquals(0, trackBytes.get());
@@ -167,12 +170,15 @@ public class TestDocValues extends Lucen
Type.FIXED_INTS_64, Type.FIXED_INTS_64,
Type.FIXED_INTS_64, Type.VAR_INTS, Type.VAR_INTS,
Type.VAR_INTS, };
+ DocValueHolder valueHolder = new DocValueHolder();
for (int i = 0; i < minMax.length; i++) {
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.VAR_INTS, newIOContext(random));
- w.add(0, minMax[i][0]);
- w.add(1, minMax[i][1]);
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.VAR_INTS, newIOContext(random));
+ valueHolder.intValue = minMax[i][0];
+ w.add(0, valueHolder);
+ valueHolder.intValue = minMax[i][1];
+ w.add(1, valueHolder);
w.finish(2);
assertEquals(0, trackBytes.get());
DocValues r = Ints.getValues(dir, "test", 2, Type.VAR_INTS, newIOContext(random));
@@ -200,12 +206,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt8Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
byte[] sourceArray = new byte[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_8, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_8, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, (long) sourceArray[i]);
+ valueHolder.intValue = (long) sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_8, newIOContext(random));
@@ -221,12 +229,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt16Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
short[] sourceArray = new short[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_16, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_16, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, (long) sourceArray[i]);
+ valueHolder.intValue = (long) sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_16, newIOContext(random));
@@ -242,12 +252,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt64Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
long[] sourceArray = new long[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_64, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_64, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, sourceArray[i]);
+ valueHolder.intValue = sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_64, newIOContext(random));
@@ -263,12 +275,14 @@ public class TestDocValues extends Lucen
}
public void testGetInt32Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
int[] sourceArray = new int[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_32, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, Type.FIXED_INTS_32, newIOContext(random));
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, (long) sourceArray[i]);
+ valueHolder.intValue = (long) sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Ints.getValues(dir, "test", sourceArray.length, Type.FIXED_INTS_32, newIOContext(random));
@@ -284,12 +298,14 @@ public class TestDocValues extends Lucen
}
public void testGetFloat32Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
float[] sourceArray = new float[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_32);
+ DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_32);
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, sourceArray[i]);
+ valueHolder.floatValue = sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Floats.getValues(dir, "test", 3, newIOContext(random), Type.FLOAT_32);
@@ -305,12 +321,14 @@ public class TestDocValues extends Lucen
}
public void testGetFloat64Array() throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
double[] sourceArray = new double[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_64);
+ DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), Type.FLOAT_64);
for (int i = 0; i < sourceArray.length; i++) {
- w.add(i, sourceArray[i]);
+ valueHolder.floatValue = sourceArray[i];
+ w.add(i, valueHolder);
}
w.finish(sourceArray.length);
DocValues r = Floats.getValues(dir, "test", 3, newIOContext(random), Type.FLOAT_64);
@@ -326,17 +344,18 @@ public class TestDocValues extends Lucen
}
private void testInts(Type type, int maxBit) throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
long maxV = 1;
final int NUM_VALUES = 333 + random.nextInt(333);
final long[] values = new long[NUM_VALUES];
for (int rx = 1; rx < maxBit; rx++, maxV *= 2) {
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Ints.getWriter(dir, "test", trackBytes, type, newIOContext(random));
+ DocValuesConsumer w = Ints.getWriter(dir, "test", trackBytes, type, newIOContext(random));
for (int i = 0; i < NUM_VALUES; i++) {
final long v = random.nextLong() % (1 + maxV);
- values[i] = v;
- w.add(i, v);
+ valueHolder.intValue = values[i] = v;
+ w.add(i, valueHolder);
}
final int additionalDocs = 1 + random.nextInt(9);
w.finish(NUM_VALUES + additionalDocs);
@@ -362,16 +381,17 @@ public class TestDocValues extends Lucen
}
private void runTestFloats(Type type, double delta) throws IOException {
+ DocValueHolder valueHolder = new DocValueHolder();
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), type);
+ DocValuesConsumer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), type);
final int NUM_VALUES = 777 + random.nextInt(777);;
final double[] values = new double[NUM_VALUES];
for (int i = 0; i < NUM_VALUES; i++) {
final double v = type == Type.FLOAT_32 ? random.nextFloat() : random
.nextDouble();
- values[i] = v;
- w.add(i, v);
+ valueHolder.floatValue = values[i] = v;
+ w.add(i, valueHolder);
}
final int additionalValues = 1 + random.nextInt(10);
w.finish(NUM_VALUES + additionalValues);
@@ -411,4 +431,31 @@ public class TestDocValues extends Lucen
return getSource(values).asSortedSource();
}
+ public static class DocValueHolder implements DocValue {
+ BytesRef bytes;
+ long intValue;
+ double floatValue;
+ Comparator<BytesRef> comp;
+ @Override
+ public BytesRef getBytes() {
+ return bytes;
+ }
+
+ @Override
+ public Comparator<BytesRef> bytesComparator() {
+ return comp;
+ }
+
+ @Override
+ public double getFloat() {
+ return floatValue;
+ }
+
+ @Override
+ public long getInt() {
+ return intValue;
+ }
+
+ }
+
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDoc.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDoc.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDoc.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDoc.java Sat Jan 7 18:23:08 2012
@@ -26,8 +26,6 @@ import java.io.StringWriter;
import java.util.LinkedList;
import java.util.Collection;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
@@ -45,11 +43,6 @@ import org.apache.lucene.util._TestUtil;
/** JUnit adaptation of an older test case DocTest. */
public class TestDoc extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run (new TestSuite(TestDoc.class));
- }
-
private File workDir;
private File indexDir;
private LinkedList<File> files;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocValuesIndexing.java Sat Jan 7 18:23:08 2012
@@ -21,12 +21,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Comparator;
import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
-import org.apache.lucene.codecs.PerDocProducer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.DocValuesField;
@@ -34,6 +39,7 @@ import org.apache.lucene.document.String
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
@@ -48,6 +54,7 @@ import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.BytesRefHash;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -417,11 +424,130 @@ public class TestDocValuesIndexing exten
w.close();
d.close();
}
+
+ public void testGetArrayNumerics() throws CorruptIndexException, IOException {
+ Directory d = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
+ IndexWriter w = new IndexWriter(d, cfg);
+ final int numValues = 50 + atLeast(10);
+ final List<Type> numVariantList = new ArrayList<Type>(NUMERICS);
+ Collections.shuffle(numVariantList, random);
+ for (Type val : numVariantList) {
+ indexValues(w, numValues, val, numVariantList,
+ false, 7);
+ IndexReader r = IndexReader.open(w, true);
+ DocValues docValues = getDocValues(r, val.name());
+ assertNotNull(docValues);
+ // make sure we don't get a direct source since they don't support getArray()
+ Source source = docValues.getSource();
+
+ switch (source.type()) {
+ case FIXED_INTS_8:
+ {
+ assertTrue(source.hasArray());
+ byte[] values = (byte[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((long)values[i], source.getInt(i));
+ }
+ }
+ break;
+ case FIXED_INTS_16:
+ {
+ assertTrue(source.hasArray());
+ short[] values = (short[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((long)values[i], source.getInt(i));
+ }
+ }
+ break;
+ case FIXED_INTS_32:
+ {
+ assertTrue(source.hasArray());
+ int[] values = (int[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((long)values[i], source.getInt(i));
+ }
+ }
+ break;
+ case FIXED_INTS_64:
+ {
+ assertTrue(source.hasArray());
+ long[] values = (long[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals(values[i], source.getInt(i));
+ }
+ }
+ break;
+ case VAR_INTS:
+ assertFalse(source.hasArray());
+ break;
+ case FLOAT_32:
+ {
+ assertTrue(source.hasArray());
+ float[] values = (float[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals((double)values[i], source.getFloat(i), 0.0d);
+ }
+ }
+ break;
+ case FLOAT_64:
+ {
+ assertTrue(source.hasArray());
+ double[] values = (double[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ assertEquals(values[i], source.getFloat(i), 0.0d);
+ }
+ }
+ break;
+ default:
+ fail("unexpected value " + source.type());
+ }
+ r.close();
+ }
+ w.close();
+ d.close();
+ }
+
+ public void testGetArrayBytes() throws CorruptIndexException, IOException {
+ Directory d = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ IndexWriter w = new IndexWriter(d, cfg);
+ final int numValues = 50 + atLeast(10);
+ // only single byte fixed straight supports getArray()
+ indexValues(w, numValues, Type.BYTES_FIXED_STRAIGHT, null, false, 1);
+ IndexReader r = IndexReader.open(w, true);
+ DocValues docValues = getDocValues(r, Type.BYTES_FIXED_STRAIGHT.name());
+ assertNotNull(docValues);
+ // make sure we don't get a direct source since they don't support
+ // getArray()
+ Source source = docValues.getSource();
+
+ switch (source.type()) {
+ case BYTES_FIXED_STRAIGHT: {
+ BytesRef ref = new BytesRef();
+ assertTrue(source.hasArray());
+ byte[] values = (byte[]) source.getArray();
+ for (int i = 0; i < numValues; i++) {
+ source.getBytes(i, ref);
+ assertEquals(1, ref.length);
+ assertEquals(values[i], ref.bytes[ref.offset]);
+ }
+ }
+ break;
+ default:
+ fail("unexpected value " + source.type());
+ }
+ r.close();
+ w.close();
+ d.close();
+ }
private DocValues getDocValues(IndexReader reader, String field) throws IOException {
return MultiDocValues.getDocValues(reader, field);
}
+ @SuppressWarnings("fallthrough")
private Source getSource(DocValues values) throws IOException {
// getSource uses cache internally
switch(random.nextInt(5)) {
@@ -430,7 +556,9 @@ public class TestDocValuesIndexing exten
case 2:
return values.getDirectSource();
case 1:
- return values.getSource();
+ if(values.type() == Type.BYTES_VAR_SORTED || values.type() == Type.BYTES_FIXED_SORTED) {
+ return values.getSource().asSortedSource();
+ }
default:
return values.getSource();
}
@@ -588,4 +716,100 @@ public class TestDocValuesIndexing exten
r.close();
d.close();
}
+
+ public void testSortedBytes() throws IOException {
+ Type[] types = new Type[] { Type.BYTES_FIXED_SORTED, Type.BYTES_VAR_SORTED };
+ for (Type type : types) {
+ boolean fixed = type == Type.BYTES_FIXED_SORTED;
+ final Directory d = newDirectory();
+ IndexWriterConfig cfg = newIndexWriterConfig(TEST_VERSION_CURRENT,
+ new MockAnalyzer(random));
+ IndexWriter w = new IndexWriter(d, cfg);
+ Comparator<BytesRef> comp = BytesRef.getUTF8SortedAsUnicodeComparator();
+ int numDocs = atLeast(100);
+ BytesRefHash hash = new BytesRefHash();
+ Map<String, String> docToString = new HashMap<String, String>();
+ int len = 1 + random.nextInt(50);
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ doc.add(newField("id", "" + i, TextField.TYPE_STORED));
+ DocValuesField f = new DocValuesField("field");
+ String string =fixed ? _TestUtil.randomFixedByteLengthUnicodeString(random,
+ len) : _TestUtil.randomRealisticUnicodeString(random, 1, len);
+ hash.add(new BytesRef(string));
+ docToString.put("" + i, string);
+
+ f.setBytes(new BytesRef(string), type, comp);
+ doc.add(f);
+ w.addDocument(doc);
+ }
+ if (rarely()) {
+ w.commit();
+ }
+ int numDocsNoValue = atLeast(10);
+ for (int i = 0; i < numDocsNoValue; i++) {
+ Document doc = new Document();
+ doc.add(newField("id", "noValue", TextField.TYPE_STORED));
+ w.addDocument(doc);
+ }
+ BytesRef bytesRef = new BytesRef(fixed ? len : 0);
+ bytesRef.offset = 0;
+ bytesRef.length = fixed ? len : 0;
+ hash.add(bytesRef); // add empty value for the gaps
+ if (rarely()) {
+ w.commit();
+ }
+ for (int i = 0; i < numDocs; i++) {
+ Document doc = new Document();
+ String id = "" + i + numDocs;
+ doc.add(newField("id", id, TextField.TYPE_STORED));
+ DocValuesField f = new DocValuesField("field");
+ String string = fixed ? _TestUtil.randomFixedByteLengthUnicodeString(random,
+ len) : _TestUtil.randomRealisticUnicodeString(random, 1, len);
+ hash.add(new BytesRef(string));
+ docToString.put(id, string);
+ f.setBytes(new BytesRef(string), type, comp);
+ doc.add(f);
+ w.addDocument(doc);
+ }
+ w.commit();
+ IndexReader reader = w.getReader();
+ DocValues docValues = MultiDocValues.getDocValues(reader, "field");
+ Source source = getSource(docValues);
+ SortedSource asSortedSource = source.asSortedSource();
+ int[] sort = hash.sort(comp);
+ BytesRef expected = new BytesRef();
+ BytesRef actual = new BytesRef();
+ assertEquals(hash.size(), asSortedSource.getValueCount());
+ for (int i = 0; i < hash.size(); i++) {
+ hash.get(sort[i], expected);
+ asSortedSource.getByOrd(i, actual);
+ assertEquals(expected.utf8ToString(), actual.utf8ToString());
+ int ord = asSortedSource.getByValue(expected, actual);
+ assertEquals(i, ord);
+ }
+ reader = new SlowMultiReaderWrapper(reader);
+ Set<Entry<String, String>> entrySet = docToString.entrySet();
+
+ for (Entry<String, String> entry : entrySet) {
+ int docId = docId(reader, new Term("id", entry.getKey()));
+ expected.copyChars(entry.getValue());
+ assertEquals(expected, asSortedSource.getBytes(docId, actual));
+ }
+
+ reader.close();
+ w.close();
+ d.close();
+ }
+ }
+
+ public int docId(IndexReader reader, Term term) throws IOException {
+ int docFreq = reader.docFreq(term);
+ assertEquals(1, docFreq);
+ DocsEnum termDocsEnum = reader.termDocsEnum(null, term.field, term.bytes, false);
+ int nextDoc = termDocsEnum.nextDoc();
+ assertEquals(DocsEnum.NO_MORE_DOCS, termDocsEnum.nextDoc());
+ return nextDoc;
+
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java Sat Jan 7 18:23:08 2012
@@ -462,9 +462,9 @@ public class TestDuelingCodecs extends L
while ((field = fieldsEnum.next()) != null) {
assertEquals(info, leftReader.hasNorms(field), rightReader.hasNorms(field));
if (leftReader.hasNorms(field)) {
- byte leftNorms[] = MultiNorms.norms(leftReader, field);
- byte rightNorms[] = MultiNorms.norms(rightReader, field);
- assertArrayEquals(info, leftNorms, rightNorms);
+ DocValues leftNorms = MultiDocValues.getNormDocValues(leftReader, field);
+ DocValues rightNorms = MultiDocValues.getNormDocValues(rightReader, field);
+ assertDocValues(leftNorms, rightNorms);
}
}
}
@@ -528,15 +528,19 @@ public class TestDuelingCodecs extends L
for (String field : leftValues) {
DocValues leftDocValues = MultiDocValues.getDocValues(leftReader, field);
DocValues rightDocValues = MultiDocValues.getDocValues(rightReader, field);
- assertNotNull(info, leftDocValues);
- assertNotNull(info, rightDocValues);
- assertEquals(info, leftDocValues.type(), rightDocValues.type());
- assertEquals(info, leftDocValues.getValueSize(), rightDocValues.getValueSize());
- assertDocValuesSource(leftDocValues.getDirectSource(), rightDocValues.getDirectSource());
- assertDocValuesSource(leftDocValues.getSource(), rightDocValues.getSource());
+ assertDocValues(leftDocValues, rightDocValues);
}
}
+ public void assertDocValues(DocValues leftDocValues, DocValues rightDocValues) throws Exception {
+ assertNotNull(info, leftDocValues);
+ assertNotNull(info, rightDocValues);
+ assertEquals(info, leftDocValues.type(), rightDocValues.type());
+ assertEquals(info, leftDocValues.getValueSize(), rightDocValues.getValueSize());
+ assertDocValuesSource(leftDocValues.getDirectSource(), rightDocValues.getDirectSource());
+ assertDocValuesSource(leftDocValues.getSource(), rightDocValues.getSource());
+ }
+
/**
* checks source API
*/
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Sat Jan 7 18:23:08 2012
@@ -509,6 +509,7 @@ public class TestIndexReader extends Luc
writer.addDocument(doc);
}
+ // TODO: maybe this can reuse the logic of test dueling codecs?
public static void assertIndexEquals(IndexReader index1, IndexReader index2) throws IOException {
assertEquals("IndexReaders have different values for numDocs.", index1.numDocs(), index2.numDocs());
assertEquals("IndexReaders have different values for maxDoc.", index1.maxDoc(), index2.maxDoc());
@@ -531,13 +532,16 @@ public class TestIndexReader extends Luc
it1 = fields1.iterator();
while (it1.hasNext()) {
String curField = it1.next();
- byte[] norms1 = MultiNorms.norms(index1, curField);
- byte[] norms2 = MultiNorms.norms(index2, curField);
+ DocValues norms1 = MultiDocValues.getNormDocValues(index1, curField);
+ DocValues norms2 = MultiDocValues.getNormDocValues(index2, curField);
if (norms1 != null && norms2 != null)
{
- assertEquals(norms1.length, norms2.length);
- for (int i = 0; i < norms1.length; i++) {
- assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", norms1[i], norms2[i]);
+ // todo: generalize this (like TestDuelingCodecs assert)
+ byte[] b1 = (byte[]) norms1.getSource().getArray();
+ byte[] b2 = (byte[]) norms2.getSource().getArray();
+ assertEquals(b1.length, b2.length);
+ for (int i = 0; i < b1.length; i++) {
+ assertEquals("Norm different for doc " + i + " and field '" + curField + "'.", b1[i], b2[i]);
}
}
else
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Jan 7 18:23:08 2012
@@ -455,8 +455,8 @@ public class TestIndexWriter extends Luc
IndexReader reader = IndexReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
- ScoreDoc[] hits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1000).scoreDocs;
- assertEquals(n*100, hits.length);
+ int totalHits = searcher.search(new TermQuery(new Term("field", "aaa")), null, 1).totalHits;
+ assertEquals(n*100, totalHits);
reader.close();
dir.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java Sat Jan 7 18:23:08 2012
@@ -75,7 +75,7 @@ public class TestMaxTermFrequency extend
}
public void test() throws Exception {
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
+ byte fooNorms[] = (byte[])MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(expected.get(i).intValue(), fooNorms[i] & 0xff);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestNorms.java Sat Jan 7 18:23:08 2012
@@ -82,11 +82,11 @@ public class TestNorms extends LuceneTes
IndexReader reader = writer.getReader();
writer.close();
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
+ byte fooNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(0, fooNorms[i]);
- byte barNorms[] = MultiNorms.norms(reader, "bar");
+ byte barNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "bar").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(1, barNorms[i]);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestOmitNorms.java Sat Jan 7 18:23:08 2012
@@ -289,12 +289,15 @@ public class TestOmitNorms extends Lucen
}
IndexReader ir1 = riw.getReader();
- byte[] norms1 = MultiNorms.norms(ir1, field);
+ // todo: generalize
+ DocValues dv1 = MultiDocValues.getNormDocValues(ir1, field);
+ byte[] norms1 = dv1 == null ? null : (byte[]) dv1.getSource().getArray();
// fully merge and validate MultiNorms against single segment.
riw.forceMerge(1);
IndexReader ir2 = riw.getReader();
- byte[] norms2 = ir2.getSequentialSubReaders()[0].norms(field);
+ DocValues dv2 = ir2.getSequentialSubReaders()[0].normValues(field);
+ byte[] norms2 = dv2 == null ? null : (byte[]) dv2.getSource().getArray();
assertArrayEquals(norms1, norms2);
ir1.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java Sat Jan 7 18:23:08 2012
@@ -172,7 +172,7 @@ public class TestSegmentReader extends L
assertEquals(reader.hasNorms(f.name()), !DocHelper.noNorms.containsKey(f.name()));
if (!reader.hasNorms(f.name())) {
// test for norms of null
- byte [] norms = MultiNorms.norms(reader, f.name());
+ DocValues norms = MultiDocValues.getNormDocValues(reader, f.name());
assertNull(norms);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTypePromotion.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTypePromotion.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTypePromotion.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestTypePromotion.java Sat Jan 7 18:23:08 2012
@@ -100,14 +100,12 @@ public class TestTypePromotion extends L
randomValueType(types, random), values, num_1 + num_2, num_3);
writer_2.commit();
writer_2.close();
- if (random.nextBoolean()) {
+ if (rarely()) {
writer.addIndexes(dir_2);
} else {
// do a real merge here
IndexReader open = IndexReader.open(dir_2);
- // we cannot use SlowMR for sorted bytes, because it returns a null sortedsource
- boolean useSlowMRWrapper = types != SORTED_BYTES && random.nextBoolean();
- writer.addIndexes(useSlowMRWrapper ? new SlowMultiReaderWrapper(open) : open);
+ writer.addIndexes(new SlowMultiReaderWrapper(open));
open.close();
}
dir_2.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java Sat Jan 7 18:23:08 2012
@@ -73,7 +73,7 @@ public class TestUniqueTermCount extends
}
public void test() throws Exception {
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
+ byte fooNorms[] = (byte[])MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
for (int i = 0; i < reader.maxDoc(); i++)
assertEquals(expected.get(i).intValue(), fooNorms[i] & 0xff);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java Sat Jan 7 18:23:08 2012
@@ -30,13 +30,11 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocValues.Source;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TermContext;
/**
* Tests the use of indexdocvalues in scoring.
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java Sat Jan 7 18:23:08 2012
@@ -24,7 +24,7 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.MultiNorms;
+import org.apache.lucene.index.MultiDocValues;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
@@ -74,8 +74,9 @@ public class TestSimilarityProvider exte
public void testBasics() throws Exception {
// sanity check of norms writer
- byte fooNorms[] = MultiNorms.norms(reader, "foo");
- byte barNorms[] = MultiNorms.norms(reader, "bar");
+ // TODO: generalize
+ byte fooNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "foo").getSource().getArray();
+ byte barNorms[] = (byte[]) MultiDocValues.getNormDocValues(reader, "bar").getSource().getArray();
for (int i = 0; i < fooNorms.length; i++) {
assertFalse(fooNorms[i] == barNorms[i]);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/search/TestSort.java Sat Jan 7 18:23:08 2012
@@ -258,7 +258,7 @@ public class TestSort extends LuceneTest
//System.out.println(writer.getSegmentCount());
writer.close();
IndexReader reader = IndexReader.open(indexStore);
- return new IndexSearcher (reader);
+ return newSearcher(reader);
}
public String getRandomNumberString(int num, int low, int high) {
@@ -1210,35 +1210,11 @@ public class TestSort extends LuceneTest
assertMatches( null, searcher, query, sort, expectedResult );
}
- private static boolean hasSlowMultiReaderWrapper(IndexReader r) {
- if (r instanceof SlowMultiReaderWrapper) {
- return true;
- } else {
- IndexReader[] subReaders = r.getSequentialSubReaders();
- if (subReaders != null) {
- for (IndexReader subReader : subReaders) {
- if (hasSlowMultiReaderWrapper(subReader)) {
- return true;
- }
- }
- }
- }
- return false;
- }
// make sure the documents returned by the search match the expected list
private void assertMatches(String msg, IndexSearcher searcher, Query query, Sort sort,
String expectedResult) throws IOException {
- for(SortField sortField : sort.getSort()) {
- if (sortField.getUseIndexValues() && sortField.getType() == SortField.Type.STRING) {
- if (hasSlowMultiReaderWrapper(searcher.getIndexReader())) {
- // Cannot use STRING DocValues sort with SlowMultiReaderWrapper
- return;
- }
- }
- }
-
//ScoreDoc[] result = searcher.search (query, null, 1000, sort).scoreDocs;
TopDocs hits = searcher.search(query, null, Math.max(1, expectedResult.length()), sort);
ScoreDoc[] result = hits.scoreDocs;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/util/TestSortedVIntList.java Sat Jan 7 18:23:08 2012
@@ -20,16 +20,10 @@ package org.apache.lucene.util;
import java.io.IOException;
import java.util.BitSet;
-import junit.framework.TestSuite;
-import junit.textui.TestRunner;
import org.apache.lucene.search.DocIdSetIterator;
public class TestSortedVIntList extends LuceneTestCase {
- /** Main for running test case by itself. */
- public static void main(String args[]) {
- TestRunner.run(new TestSuite(TestSortedVIntList.class));
- }
void tstIterator (
SortedVIntList vintList,
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Sat Jan 7 18:23:08 2012
@@ -153,12 +153,15 @@ public final class SynonymFilter extends
// Holds pending output synonyms for one future position:
private static class PendingOutputs {
CharsRef[] outputs;
+ int[] endOffsets;
int upto;
int count;
int posIncr = 1;
+ int lastEndOffset;
public PendingOutputs() {
outputs = new CharsRef[1];
+ endOffsets = new int[1];
}
public void reset() {
@@ -168,6 +171,7 @@ public final class SynonymFilter extends
public CharsRef pullNext() {
assert upto < count;
+ lastEndOffset = endOffsets[upto];
final CharsRef result = outputs[upto++];
posIncr = 0;
if (upto == count) {
@@ -176,16 +180,29 @@ public final class SynonymFilter extends
return result;
}
- public void add(char[] output, int offset, int len) {
+ public int getLastEndOffset() {
+ return lastEndOffset;
+ }
+
+ public void add(char[] output, int offset, int len, int endOffset) {
if (count == outputs.length) {
final CharsRef[] next = new CharsRef[ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_OBJECT_REF)];
System.arraycopy(outputs, 0, next, 0, count);
outputs = next;
}
+ if (count == endOffsets.length) {
+ final int[] next = new int[ArrayUtil.oversize(1+count, RamUsageEstimator.NUM_BYTES_INT)];
+ System.arraycopy(endOffsets, 0, next, 0, count);
+ endOffsets = next;
+ }
if (outputs[count] == null) {
outputs[count] = new CharsRef();
}
outputs[count].copyChars(output, offset, len);
+ // endOffset can be -1, in which case we should simply
+ // use the endOffset of the input token, or X >= 0, in
+ // which case we use X as the endOffset for this output
+ endOffsets[count] = endOffset;
count++;
}
};
@@ -281,6 +298,7 @@ public final class SynonymFilter extends
// Holds the longest match we've seen so far:
BytesRef matchOutput = null;
int matchInputLength = 0;
+ int matchEndOffset = -1;
BytesRef pendingOutput = fst.outputs.getNoOutput();
fst.getFirstArc(scratchArc);
@@ -297,6 +315,8 @@ public final class SynonymFilter extends
final int bufferLen;
//System.out.println(" cycle nextRead=" + curNextRead + " nextWrite=" + nextWrite);
+ int inputEndOffset = 0;
+
if (curNextRead == nextWrite) {
// We used up our lookahead buffer of input tokens
@@ -317,6 +337,7 @@ public final class SynonymFilter extends
final PendingInput input = futureInputs[nextWrite];
input.startOffset = offsetAtt.startOffset();
input.endOffset = offsetAtt.endOffset();
+ inputEndOffset = input.endOffset;
//System.out.println(" new token=" + new String(buffer, 0, bufferLen));
if (nextRead != nextWrite) {
capture();
@@ -335,6 +356,7 @@ public final class SynonymFilter extends
// Still in our lookahead
buffer = futureInputs[curNextRead].term.chars;
bufferLen = futureInputs[curNextRead].term.length;
+ inputEndOffset = futureInputs[curNextRead].endOffset;
//System.out.println(" old token=" + new String(buffer, 0, bufferLen));
}
@@ -360,6 +382,7 @@ public final class SynonymFilter extends
if (scratchArc.isFinal()) {
matchOutput = fst.outputs.add(pendingOutput, scratchArc.nextFinalOutput);
matchInputLength = tokenCount;
+ matchEndOffset = inputEndOffset;
//System.out.println(" found matchLength=" + matchInputLength + " output=" + matchOutput);
}
@@ -390,7 +413,7 @@ public final class SynonymFilter extends
if (matchOutput != null) {
//System.out.println(" add matchLength=" + matchInputLength + " output=" + matchOutput);
inputSkipCount = matchInputLength;
- addOutput(matchOutput, matchInputLength);
+ addOutput(matchOutput, matchInputLength, matchEndOffset);
} else if (nextRead != nextWrite) {
// Even though we had no match here, we set to 1
// because we need to skip current input token before
@@ -404,7 +427,7 @@ public final class SynonymFilter extends
}
// Interleaves all output tokens onto the futureOutputs:
- private void addOutput(BytesRef bytes, int matchInputLength) {
+ private void addOutput(BytesRef bytes, int matchInputLength, int matchEndOffset) {
bytesReader.reset(bytes.bytes, bytes.offset, bytes.length);
final int code = bytesReader.readVInt();
@@ -425,7 +448,21 @@ public final class SynonymFilter extends
// Caller is not allowed to have empty string in
// the output:
assert outputLen > 0: "output contains empty string: " + scratchChars;
- futureOutputs[outputUpto].add(scratchChars.chars, lastStart, outputLen);
+ final int endOffset;
+ if (chIDX == chEnd && lastStart == scratchChars.offset) {
+ // This rule had a single output token, so, we set
+ // this output's endOffset to the current
+ // endOffset (ie, endOffset of the last input
+ // token it matched):
+ endOffset = matchEndOffset;
+ } else {
+ // This rule has more than one output token; we
+ // can't pick any particular endOffset for this
+ // case, so, we inherit the endOffset for the
+ // input token which this output overlaps:
+ endOffset = -1;
+ }
+ futureOutputs[outputUpto].add(scratchChars.chars, lastStart, outputLen, endOffset);
//System.out.println(" " + new String(scratchChars.chars, lastStart, outputLen) + " outputUpto=" + outputUpto);
lastStart = 1+chIDX;
//System.out.println(" slot=" + outputUpto + " keepOrig=" + keepOrig);
@@ -507,7 +544,11 @@ public final class SynonymFilter extends
clearAttributes();
termAtt.copyBuffer(output.chars, output.offset, output.length);
typeAtt.setType(TYPE_SYNONYM);
- offsetAtt.setOffset(input.startOffset, input.endOffset);
+ int endOffset = outputs.getLastEndOffset();
+ if (endOffset == -1) {
+ endOffset = input.endOffset;
+ }
+ offsetAtt.setOffset(input.startOffset, endOffset);
posIncrAtt.setPositionIncrement(posIncr);
if (outputs.count == 0) {
// Done with the buffered input and all outputs at
Modified: lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java (original)
+++ lucene/dev/branches/solrcloud/modules/analysis/common/src/test/org/apache/lucene/analysis/synonym/TestSynonymMapFilter.java Sat Jan 7 18:23:08 2012
@@ -59,7 +59,12 @@ public class TestSynonymMapFilter extend
}
}
- // todo: we should probably refactor this guy to use/take analyzer,
+ // For the output string: separate positions with a space,
+ // and separate multiple tokens at each position with a
+ // /. If a token should have end offset != the input
+ // token's end offset then add :X to it:
+
+ // TODO: we should probably refactor this guy to use/take analyzer,
// the tests are a little messy
private void verify(String input, String output) throws Exception {
if (VERBOSE) {
@@ -73,7 +78,7 @@ public class TestSynonymMapFilter extend
while(tokensOut.incrementToken()) {
if (VERBOSE) {
- System.out.println(" incr token=" + termAtt.toString() + " posIncr=" + posIncrAtt.getPositionIncrement());
+ System.out.println(" incr token=" + termAtt.toString() + " posIncr=" + posIncrAtt.getPositionIncrement() + " startOff=" + offsetAtt.startOffset() + " endOff=" + offsetAtt.endOffset());
}
assertTrue(expectedUpto < expected.length);
@@ -85,16 +90,26 @@ public class TestSynonymMapFilter extend
if (atPos > 0) {
assertTrue(tokensOut.incrementToken());
if (VERBOSE) {
- System.out.println(" incr token=" + termAtt.toString() + " posIncr=" + posIncrAtt.getPositionIncrement());
+ System.out.println(" incr token=" + termAtt.toString() + " posIncr=" + posIncrAtt.getPositionIncrement() + " startOff=" + offsetAtt.startOffset() + " endOff=" + offsetAtt.endOffset());
}
}
- assertEquals(termAtt, expectedAtPos[atPos]);
+ final int colonIndex = expectedAtPos[atPos].indexOf(':');
+ final String expectedToken;
+ final int expectedEndOffset;
+ if (colonIndex != -1) {
+ expectedToken = expectedAtPos[atPos].substring(0, colonIndex);
+ expectedEndOffset = Integer.parseInt(expectedAtPos[atPos].substring(1+colonIndex));
+ } else {
+ expectedToken = expectedAtPos[atPos];
+ expectedEndOffset = endOffset;
+ }
+ assertEquals(expectedToken, termAtt.toString());
assertEquals(atPos == 0 ? 1 : 0,
posIncrAtt.getPositionIncrement());
// start/end offset of all tokens at same pos should
// be the same:
assertEquals(startOffset, offsetAtt.startOffset());
- assertEquals(endOffset, offsetAtt.endOffset());
+ assertEquals(expectedEndOffset, offsetAtt.endOffset());
}
}
tokensOut.end();
@@ -112,6 +127,7 @@ public class TestSynonymMapFilter extend
add("b c", "dog collar", true);
add("c d", "dog harness holder extras", true);
add("m c e", "dog barks loudly", false);
+ add("i j k", "feep", true);
add("e f", "foo bar", false);
add("e f", "baz bee", false);
@@ -148,6 +164,9 @@ public class TestSynonymMapFilter extend
// two outputs for same input
verify("e f", "foo/baz bar/bee");
+ // verify multi-word / single-output offsets:
+ verify("g i j k g", "g i/feep:7 j k g");
+
// mixed keepOrig true/false:
verify("a m c e x", "a/foo dog barks loudly x");
verify("c d m c e x", "c/dog d/harness holder/dog extras/barks loudly x");
@@ -241,6 +260,10 @@ public class TestSynonymMapFilter extend
} else {
outputs[matchIDX] = outputs[matchIDX] + "/" + synOutputs[synUpto++];
}
+ if (synOutputs.length == 1) {
+ // Add endOffset
+ outputs[matchIDX] = outputs[matchIDX] + ":" + ((inputIDX*2) + syn.in.length());
+ }
}
}
}
@@ -663,4 +686,24 @@ public class TestSynonymMapFilter extend
new String[] { "zoo", "zoo", "zoo", "$", "zoo", "zoo", "zoo" },
new int[] { 1, 0, 1, 1, 1, 0, 1 });
}
+
+ public void testMultiwordOffsets() throws Exception {
+ b = new SynonymMap.Builder(true);
+ final boolean keepOrig = true;
+ add("national hockey league", "nhl", keepOrig);
+ final SynonymMap map = b.build();
+ Analyzer a = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
+ return new TokenStreamComponents(tokenizer, new SynonymFilter(tokenizer, map, true));
+ }
+ };
+
+ assertAnalyzesTo(a, "national hockey league",
+ new String[] { "national", "nhl", "hockey", "league" },
+ new int[] { 0, 0, 9, 16 },
+ new int[] { 8, 22, 15, 22 },
+ new int[] { 1, 0, 1, 1 });
+ }
}
Modified: lucene/dev/branches/solrcloud/modules/benchmark/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/benchmark/build.xml?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/benchmark/build.xml (original)
+++ lucene/dev/branches/solrcloud/modules/benchmark/build.xml Sat Jan 7 18:23:08 2012
@@ -154,6 +154,7 @@
<pathelement path="${analyzers-common.jar}"/>
<pathelement path="${queryparser.jar}"/>
<pathelement path="${facet.jar}"/>
+ <fileset dir="${common.dir}/../modules/analysis/icu/lib" includes="icu4j*.jar"/>
<path refid="base.classpath"/>
<fileset dir="lib">
<include name="**/*.jar"/>
Modified: lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java (original)
+++ lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishContentSource.java Sat Jan 7 18:23:08 2012
@@ -17,10 +17,11 @@ package org.apache.lucene.benchmark.byTa
* limitations under the License.
*/
-import org.apache.lucene.util.English;
-
import java.io.IOException;
import java.util.Date;
+import java.util.Locale;
+
+import com.ibm.icu.text.RuleBasedNumberFormat;
/**
* Creates documents whose content is a <code>long</code> number starting from
@@ -32,7 +33,10 @@ public class LongToEnglishContentSource
@Override
public void close() throws IOException {
}
-
+
+ // TODO: we could take param to specify locale...
+ private final RuleBasedNumberFormat rnbf = new RuleBasedNumberFormat(Locale.ENGLISH,
+ RuleBasedNumberFormat.SPELLOUT);
@Override
public synchronized DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException {
docData.clear();
@@ -46,7 +50,8 @@ public class LongToEnglishContentSource
++counter;
}
}
- docData.setBody(English.longToEnglish(curCounter));
+
+ docData.setBody(rnbf.format(curCounter));
docData.setName("doc_" + String.valueOf(curCounter));
docData.setTitle("title_" + String.valueOf(curCounter));
docData.setDate(new Date());
Modified: lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java (original)
+++ lucene/dev/branches/solrcloud/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LongToEnglishQueryMaker.java Sat Jan 7 18:23:08 2012
@@ -1,3 +1,5 @@
+package org.apache.lucene.benchmark.byTask.feeds;
+
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,7 +17,7 @@
* limitations under the License.
*/
-package org.apache.lucene.benchmark.byTask.feeds;
+import java.util.Locale;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
@@ -23,9 +25,8 @@ import org.apache.lucene.benchmark.byTas
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.Query;
-import org.apache.lucene.util.English;
import org.apache.lucene.util.Version;
-
+import com.ibm.icu.text.RuleBasedNumberFormat;
/**
*
@@ -35,13 +36,16 @@ public class LongToEnglishQueryMaker imp
long counter = Long.MIN_VALUE + 10;
protected QueryParser parser;
+ // TODO: we could take param to specify locale...
+ private final RuleBasedNumberFormat rnbf = new RuleBasedNumberFormat(Locale.ENGLISH,
+ RuleBasedNumberFormat.SPELLOUT);
+
public Query makeQuery(int size) throws Exception {
throw new UnsupportedOperationException();
}
public synchronized Query makeQuery() throws Exception {
-
- return parser.parse("" + English.longToEnglish(getNextCounter()) + "");
+ return parser.parse("" + rnbf.format(getNextCounter()) + "");
}
private synchronized long getNextCounter() {
Modified: lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java?rev=1228693&r1=1228692&r2=1228693&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java (original)
+++ lucene/dev/branches/solrcloud/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java Sat Jan 7 18:23:08 2012
@@ -17,6 +17,7 @@
package org.apache.lucene.queries.function.valuesource;
+import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.queries.function.FunctionValues;
import org.apache.lucene.queries.function.ValueSource;
@@ -56,10 +57,13 @@ public class NormValueSource extends Val
throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
}
final TFIDFSimilarity similarity = (TFIDFSimilarity) sim;
- final byte[] norms = readerContext.reader.norms(field);
- if (norms == null) {
+ DocValues dv = readerContext.reader.normValues(field);
+
+ if (dv == null) {
return new ConstDoubleDocValues(0.0, this);
}
+
+ final byte[] norms = (byte[]) dv.getSource().getArray();
return new FloatDocValues(this) {
@Override