You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2013/06/12 16:48:48 UTC
svn commit: r1492231 [2/2] - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/el/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/
lucene/analysis/common/src/java...
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java?rev=1492231&r1=1492230&r2=1492231&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/ngram/NGramTokenizerTest.java Wed Jun 12 14:48:47 2013
@@ -18,9 +18,12 @@ package org.apache.lucene.analysis.ngram
*/
+import static org.apache.lucene.analysis.ngram.NGramTokenizerTest.isTokenChar;
+
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@@ -115,23 +118,74 @@ public class NGramTokenizerTest extends
checkRandomData(random(), a, 50*RANDOM_MULTIPLIER, 1027, false, false);
}
- private void testNGrams(int minGram, int maxGram, int length) throws IOException {
+ private static void testNGrams(int minGram, int maxGram, int length, final String nonTokenChars) throws IOException {
final String s = RandomStrings.randomAsciiOfLength(random(), length);
- final TokenStream grams = new NGramTokenizer(TEST_VERSION_CURRENT, new StringReader(s), minGram, maxGram);
+ testNGrams(minGram, maxGram, s, nonTokenChars);
+ }
+
+ private static void testNGrams(int minGram, int maxGram, String s, String nonTokenChars) throws IOException {
+ testNGrams(minGram, maxGram, s, nonTokenChars, false);
+ }
+
+ static int[] toCodePoints(CharSequence s) {
+ final int[] codePoints = new int[Character.codePointCount(s, 0, s.length())];
+ for (int i = 0, j = 0; i < s.length(); ++j) {
+ codePoints[j] = Character.codePointAt(s, i);
+ i += Character.charCount(codePoints[j]);
+ }
+ return codePoints;
+ }
+
+ static boolean isTokenChar(String nonTokenChars, int codePoint) {
+ for (int i = 0; i < nonTokenChars.length(); ) {
+ final int cp = nonTokenChars.codePointAt(i);
+ if (cp == codePoint) {
+ return false;
+ }
+ i += Character.charCount(cp);
+ }
+ return true;
+ }
+
+ static void testNGrams(int minGram, int maxGram, String s, final String nonTokenChars, boolean edgesOnly) throws IOException {
+ // convert the string to code points
+ final int[] codePoints = toCodePoints(s);
+ final int[] offsets = new int[codePoints.length + 1];
+ for (int i = 0; i < codePoints.length; ++i) {
+ offsets[i+1] = offsets[i] + Character.charCount(codePoints[i]);
+ }
+ final TokenStream grams = new NGramTokenizer(TEST_VERSION_CURRENT, new StringReader(s), minGram, maxGram, edgesOnly) {
+ @Override
+ protected boolean isTokenChar(int chr) {
+ return nonTokenChars.indexOf(chr) < 0;
+ }
+ };
final CharTermAttribute termAtt = grams.addAttribute(CharTermAttribute.class);
final PositionIncrementAttribute posIncAtt = grams.addAttribute(PositionIncrementAttribute.class);
final PositionLengthAttribute posLenAtt = grams.addAttribute(PositionLengthAttribute.class);
final OffsetAttribute offsetAtt = grams.addAttribute(OffsetAttribute.class);
grams.reset();
- for (int start = 0; start < s.length(); ++start) {
- for (int end = start + minGram; end <= start + maxGram && end <= s.length(); ++end) {
+ for (int start = 0; start < codePoints.length; ++start) {
+ nextGram:
+ for (int end = start + minGram; end <= start + maxGram && end <= codePoints.length; ++end) {
+ if (edgesOnly && start > 0 && isTokenChar(nonTokenChars, codePoints[start - 1])) {
+ // not on an edge
+ continue nextGram;
+ }
+ for (int j = start; j < end; ++j) {
+ if (!isTokenChar(nonTokenChars, codePoints[j])) {
+ continue nextGram;
+ }
+ }
assertTrue(grams.incrementToken());
- assertEquals(s.substring(start, end), termAtt.toString());
+ assertArrayEquals(Arrays.copyOfRange(codePoints, start, end), toCodePoints(termAtt));
assertEquals(1, posIncAtt.getPositionIncrement());
- assertEquals(start, offsetAtt.startOffset());
- assertEquals(end, offsetAtt.endOffset());
+ assertEquals(1, posLenAtt.getPositionLength());
+ assertEquals(offsets[start], offsetAtt.startOffset());
+ assertEquals(offsets[end], offsetAtt.endOffset());
}
}
+ assertFalse(grams.incrementToken());
grams.end();
assertEquals(s.length(), offsetAtt.startOffset());
assertEquals(s.length(), offsetAtt.endOffset());
@@ -141,14 +195,47 @@ public class NGramTokenizerTest extends
// test sliding
final int minGram = _TestUtil.nextInt(random(), 1, 100);
final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
- testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024));
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024), "");
}
public void testLargeMaxGram() throws IOException {
// test sliding with maxGram > 1024
- final int minGram = _TestUtil.nextInt(random(), 1200, 1300);
+ final int minGram = _TestUtil.nextInt(random(), 1290, 1300);
final int maxGram = _TestUtil.nextInt(random(), minGram, 1300);
- testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024));
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 3 * 1024, 4 * 1024), "");
+ }
+
+ public void testPreTokenization() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 0, 4 * 1024), "a");
+ }
+
+ public void testHeavyPreTokenization() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ testNGrams(minGram, maxGram, _TestUtil.nextInt(random(), 0, 4 * 1024), "abcdef");
+ }
+
+ public void testFewTokenChars() throws IOException {
+ final char[] chrs = new char[_TestUtil.nextInt(random(), 4000, 5000)];
+ Arrays.fill(chrs, ' ');
+ for (int i = 0; i < chrs.length; ++i) {
+ if (random().nextFloat() < 0.1) {
+ chrs[i] = 'a';
+ }
+ }
+ final int minGram = _TestUtil.nextInt(random(), 1, 2);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 2);
+ testNGrams(minGram, maxGram, new String(chrs), " ");
+ }
+
+ public void testFullUTF8Range() throws IOException {
+ final int minGram = _TestUtil.nextInt(random(), 1, 100);
+ final int maxGram = _TestUtil.nextInt(random(), minGram, 100);
+ final String s = _TestUtil.randomUnicodeString(random(), 4 * 1024);
+ testNGrams(minGram, maxGram, s, "");
+ testNGrams(minGram, maxGram, s, "abcdef");
}
}
Modified: lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java?rev=1492231&r1=1492230&r2=1492231&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestCharacterUtils.java Wed Jun 12 14:48:47 2013
@@ -20,10 +20,13 @@ package org.apache.lucene.analysis.util;
import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
+import java.util.Arrays;
import org.apache.lucene.analysis.util.CharacterUtils.CharacterBuffer;
+import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
+import org.apache.lucene.util._TestUtil;
import org.junit.Test;
/**
@@ -32,32 +35,6 @@ import org.junit.Test;
public class TestCharacterUtils extends LuceneTestCase {
@Test
- public void testCodePointAtCharArrayInt() {
- CharacterUtils java4 = CharacterUtils.getInstance(Version.LUCENE_30);
- char[] cpAt3 = "Abc\ud801\udc1c".toCharArray();
- char[] highSurrogateAt3 = "Abc\ud801".toCharArray();
- assertEquals((int) 'A', java4.codePointAt(cpAt3, 0));
- assertEquals((int) '\ud801', java4.codePointAt(cpAt3, 3));
- assertEquals((int) '\ud801', java4.codePointAt(highSurrogateAt3, 3));
- try {
- java4.codePointAt(highSurrogateAt3, 4);
- fail("array index out of bounds");
- } catch (IndexOutOfBoundsException e) {
- }
-
- CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
- assertEquals((int) 'A', java5.codePointAt(cpAt3, 0));
- assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
- cpAt3, 3));
- assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3));
- try {
- java5.codePointAt(highSurrogateAt3, 4);
- fail("array index out of bounds");
- } catch (IndexOutOfBoundsException e) {
- }
- }
-
- @Test
public void testCodePointAtCharSequenceInt() {
CharacterUtils java4 = CharacterUtils.getInstance(Version.LUCENE_30);
String cpAt3 = "Abc\ud801\udc1c";
@@ -98,7 +75,68 @@ public class TestCharacterUtils extends
assertEquals(Character.toCodePoint('\ud801', '\udc1c'), java5.codePointAt(
cpAt3, 3, 5));
assertEquals((int) '\ud801', java5.codePointAt(highSurrogateAt3, 3, 4));
+ }
+ @Test
+ public void testCodePointCount() {
+ CharacterUtils java4 = CharacterUtils.getJava4Instance();
+ CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
+ final String s = _TestUtil.randomUnicodeString(random());
+ assertEquals(s.length(), java4.codePointCount(s));
+ assertEquals(Character.codePointCount(s, 0, s.length()), java5.codePointCount(s));
+ }
+
+ @Test
+ public void testOffsetByCodePoint() {
+ CharacterUtils java4 = CharacterUtils.getJava4Instance();
+ CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
+ for (int i = 0; i < 10; ++i) {
+ final char[] s = _TestUtil.randomUnicodeString(random()).toCharArray();
+ final int index = _TestUtil.nextInt(random(), 0, s.length);
+ final int offset = random().nextInt(7) - 3;
+ try {
+ final int o = java4.offsetByCodePoints(s, 0, s.length, index, offset);
+ assertEquals(o, index + offset);
+ } catch (IndexOutOfBoundsException e) {
+ assertTrue((index + offset) < 0 || (index + offset) > s.length);
+ }
+
+ int o;
+ try {
+ o = java5.offsetByCodePoints(s, 0, s.length, index, offset);
+ } catch (IndexOutOfBoundsException e) {
+ try {
+ Character.offsetByCodePoints(s, 0, s.length, index, offset);
+ fail();
+ } catch (IndexOutOfBoundsException e2) {
+ // OK
+ }
+ o = -1;
+ }
+ if (o >= 0) {
+ assertEquals(Character.offsetByCodePoints(s, 0, s.length, index, offset), o);
+ }
+ }
+ }
+
+ public void testConversions() {
+ CharacterUtils java4 = CharacterUtils.getJava4Instance();
+ CharacterUtils java5 = CharacterUtils.getInstance(TEST_VERSION_CURRENT);
+ testConversions(java4);
+ testConversions(java5);
+ }
+
+ private void testConversions(CharacterUtils charUtils) {
+ final char[] orig = _TestUtil.randomUnicodeString(random(), 100).toCharArray();
+ final int[] buf = new int[orig.length];
+ final char[] restored = new char[buf.length];
+ final int o1 = random().nextInt(5);
+ final int o2 = _TestUtil.nextInt(random(), 0, o1);
+ final int o3 = _TestUtil.nextInt(random(), 0, o1);
+ final int codePointCount = charUtils.toCodePoints(orig, o1, orig.length - o1, buf, o2);
+ final int charCount = charUtils.toChars(buf, o2, codePointCount, restored, o3);
+ assertEquals(orig.length - o1, charCount);
+ assertArrayEquals(Arrays.copyOfRange(orig, o1, o1 + charCount), Arrays.copyOfRange(restored, o3, o3 + charCount));
}
@Test
@@ -131,7 +169,7 @@ public class TestCharacterUtils extends
assertEquals(0, buffer.getOffset());
assertEquals(6, buffer.getLength());
assertEquals("hellow", new String(buffer.getBuffer()));
- assertTrue(instance.fill(buffer,reader));
+ assertFalse(instance.fill(buffer,reader));
assertEquals(4, buffer.getLength());
assertEquals(0, buffer.getOffset());
@@ -158,15 +196,12 @@ public class TestCharacterUtils extends
assertEquals(4, buffer.getLength());
assertEquals("123\ud801", new String(buffer.getBuffer(),
buffer.getOffset(), buffer.getLength()));
- assertTrue(instance.fill(buffer, reader));
- assertEquals(2, buffer.getLength());
- assertEquals("\ud801\udc1c", new String(buffer.getBuffer(), buffer
- .getOffset(), buffer.getLength()));
- assertTrue(instance.fill(buffer, reader));
- assertEquals(1, buffer.getLength());
- assertEquals("\ud801", new String(buffer.getBuffer(), buffer
+ assertFalse(instance.fill(buffer, reader));
+ assertEquals(3, buffer.getLength());
+ assertEquals("\ud801\udc1c\ud801", new String(buffer.getBuffer(), buffer
.getOffset(), buffer.getLength()));
assertFalse(instance.fill(buffer, reader));
+ assertEquals(0, buffer.getLength());
}
@Test
Modified: lucene/dev/branches/branch_4x/lucene/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/build.xml?rev=1492231&r1=1492230&r2=1492231&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/build.xml (original)
+++ lucene/dev/branches/branch_4x/lucene/build.xml Wed Jun 12 14:48:47 2013
@@ -183,7 +183,10 @@
<forbidden-apis internalRuntimeForbidden="true" classpathref="forbidden-apis.classpath">
<bundledSignatures name="jdk-unsafe-${javac.target}"/>
<bundledSignatures name="jdk-deprecated-${javac.target}"/>
- <signaturesFileSet file="${common.dir}/tools/forbiddenApis/executors.txt"/>
+ <signaturesFileSet dir="${common.dir}/tools/forbiddenApis">
+ <include name="executors.txt" />
+ <include name="chars.txt" />
+ </signaturesFileSet>
<fileset dir="${basedir}/build" includes="**/*.class" />
</forbidden-apis>
</target>
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java?rev=1492231&r1=1492230&r2=1492231&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java Wed Jun 12 14:48:47 2013
@@ -808,7 +808,7 @@ public final class Util {
final int charLimit = offset + length;
while(charIdx < charLimit) {
scratch.grow(intIdx+1);
- final int utf32 = Character.codePointAt(s, charIdx);
+ final int utf32 = Character.codePointAt(s, charIdx, charLimit);
scratch.ints[intIdx] = utf32;
charIdx += Character.charCount(utf32);
intIdx++;
Modified: lucene/dev/branches/branch_4x/solr/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/solr/build.xml?rev=1492231&r1=1492230&r2=1492231&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/solr/build.xml (original)
+++ lucene/dev/branches/branch_4x/solr/build.xml Wed Jun 12 14:48:47 2013
@@ -266,6 +266,7 @@
<bundledSignatures name="commons-io-unsafe-${commons-io.version}"/>
<signaturesFileSet dir="${common.dir}/tools/forbiddenApis">
<include name="executors.txt" />
+ <include name="chars.txt" />
<include name="servlet-api.txt" />
</signaturesFileSet>
<fileset dir="${basedir}/build">