You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/07/29 05:39:11 UTC
svn commit: r1152090 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/backwards/ lucene/backwards/src/test-framework/
lucene/backwards/src/test/ lucene/src/test-framework/org/apache/lucene/util/
lucene/src/test/org/apache/lucene/index/ solr/
Author: rmuir
Date: Fri Jul 29 03:39:10 2011
New Revision: 1152090
URL: http://svn.apache.org/viewvc?rev=1152090&view=rev
Log:
fix unicode violations in test, test relies upon how *invalid* utf8 is converted
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test/ (props changed)
lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1152090&r1=1152089&r2=1152090&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Fri Jul 29 03:39:10 2011
@@ -316,6 +316,48 @@ public class _TestUtil {
return sb.toString();
}
+ /** Returns random string, with a given UTF-8 byte length*/
+ public static String randomFixedByteLengthUnicodeString(Random r, int length) {
+
+ final char[] buffer = new char[length*3];
+ int bytes = length;
+ int i = 0;
+ for (; i < buffer.length && bytes != 0; i++) {
+ int t;
+ if (bytes >= 4) {
+ t = r.nextInt(5);
+ } else if (bytes >= 3) {
+ t = r.nextInt(4);
+ } else if (bytes >= 2) {
+ t = r.nextInt(2);
+ } else {
+ t = 0;
+ }
+ if (t == 0) {
+ buffer[i] = (char) r.nextInt(0x80);
+ bytes--;
+ } else if (1 == t) {
+ buffer[i] = (char) nextInt(r, 0x80, 0x7ff);
+ bytes -= 2;
+ } else if (2 == t) {
+ buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
+ bytes -= 3;
+ } else if (3 == t) {
+ buffer[i] = (char) nextInt(r, 0xe000, 0xffff);
+ bytes -= 3;
+ } else if (4 == t) {
+ // Make a surrogate pair
+ // High surrogate
+ buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff);
+ // Low surrogate
+ buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff);
+ bytes -= 4;
+ }
+
+ }
+ return new String(buffer, 0, i);
+ }
+
/** start and end are BOTH inclusive */
public static int nextInt(Random r, int start, int end) {
return start + r.nextInt(end-start+1);
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1152090&r1=1152089&r2=1152090&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java Fri Jul 29 03:39:10 2011
@@ -21,6 +21,7 @@ import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -340,8 +341,13 @@ public class TestPayloads extends Lucene
}
+ static final Charset utf8 = Charset.forName("UTF-8");
private void generateRandomData(byte[] data) {
- random.nextBytes(data);
+ // this test needs the random data to be valid unicode
+ String s = _TestUtil.randomFixedByteLengthUnicodeString(random, data.length);
+ byte b[] = s.getBytes(utf8);
+ assert b.length == data.length;
+ System.arraycopy(b, 0, data, 0, b.length);
}
private byte[] generateRandomData(int n) {
@@ -503,7 +509,9 @@ public class TestPayloads extends Lucene
int freq = tp.freq();
for (int i = 0; i < freq; i++) {
tp.nextPosition();
- assertEquals(pool.bytesToString(tp.getPayload(new byte[5], 0)), terms.term().text);
+ byte payload[] = new byte[5];
+ tp.getPayload(payload, 0);
+ assertEquals(terms.term().text, new String(payload, 0, payload.length, utf8));
}
}
tp.close();
@@ -527,7 +535,7 @@ public class TestPayloads extends Lucene
this.pool = pool;
payload = pool.get();
generateRandomData(payload);
- term = pool.bytesToString(payload);
+ term = new String(payload, 0, payload.length, utf8);
first = true;
payloadAtt = addAttribute(PayloadAttribute.class);
termAtt = addAttribute(CharTermAttribute.class);
@@ -559,18 +567,6 @@ public class TestPayloads extends Lucene
pool.add(new byte[size]);
}
}
-
- private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
-
- synchronized String bytesToString(byte[] bytes) {
- String s = new String(bytes);
- UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
- try {
- return new String(utf8Result.result, 0, utf8Result.length, "UTF-8");
- } catch (UnsupportedEncodingException uee) {
- return null;
- }
- }
synchronized byte[] get() {
return pool.remove(0);