You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/07/29 05:39:11 UTC

svn commit: r1152090 - in /lucene/dev/branches/branch_3x: ./ lucene/ lucene/backwards/ lucene/backwards/src/test-framework/ lucene/backwards/src/test/ lucene/src/test-framework/org/apache/lucene/util/ lucene/src/test/org/apache/lucene/index/ solr/

Author: rmuir
Date: Fri Jul 29 03:39:10 2011
New Revision: 1152090

URL: http://svn.apache.org/viewvc?rev=1152090&view=rev
Log:
fix unicode violations in test, test relies upon how *invalid* utf8 is converted

Modified:
    lucene/dev/branches/branch_3x/   (props changed)
    lucene/dev/branches/branch_3x/lucene/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test/   (props changed)
    lucene/dev/branches/branch_3x/lucene/backwards/src/test-framework/   (props changed)
    lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
    lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
    lucene/dev/branches/branch_3x/solr/   (props changed)

Modified: lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java?rev=1152090&r1=1152089&r2=1152090&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java Fri Jul 29 03:39:10 2011
@@ -316,6 +316,48 @@ public class _TestUtil {
     return sb.toString();
   }
 
+  /** Returns random string, with a given UTF-8 byte length*/
+  public static String randomFixedByteLengthUnicodeString(Random r, int length) {
+    
+    final char[] buffer = new char[length*3];
+    int bytes = length;
+    int i = 0;
+    for (; i < buffer.length && bytes != 0; i++) {
+      int t;
+      if (bytes >= 4) {
+        t = r.nextInt(5);
+      } else if (bytes >= 3) {
+        t = r.nextInt(4);
+      } else if (bytes >= 2) {
+        t = r.nextInt(2);
+      } else {
+        t = 0;
+      }
+      if (t == 0) {
+        buffer[i] = (char) r.nextInt(0x80);
+        bytes--;
+      } else if (1 == t) {
+        buffer[i] = (char) nextInt(r, 0x80, 0x7ff);
+        bytes -= 2;
+      } else if (2 == t) {
+        buffer[i] = (char) nextInt(r, 0x800, 0xd7ff);
+        bytes -= 3;
+      } else if (3 == t) {
+        buffer[i] = (char) nextInt(r, 0xe000, 0xffff);
+        bytes -= 3;
+      } else if (4 == t) {
+        // Make a surrogate pair
+        // High surrogate
+        buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff);
+        // Low surrogate
+        buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff);
+        bytes -= 4;
+      }
+
+    }
+    return new String(buffer, 0, i);
+  }
+
   /** start and end are BOTH inclusive */
   public static int nextInt(Random r, int start, int end) {
     return start + r.nextInt(end-start+1);

Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java?rev=1152090&r1=1152089&r2=1152090&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/index/TestPayloads.java Fri Jul 29 03:39:10 2011
@@ -21,6 +21,7 @@ import java.io.File;
 import java.io.IOException;
 import java.io.Reader;
 import java.io.UnsupportedEncodingException;
+import java.nio.charset.Charset;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -340,8 +341,13 @@ public class TestPayloads extends Lucene
         
     }
     
+    static final Charset utf8 = Charset.forName("UTF-8");
     private void generateRandomData(byte[] data) {
-        random.nextBytes(data);
+      // this test needs the random data to be valid unicode
+      String s = _TestUtil.randomFixedByteLengthUnicodeString(random, data.length);
+      byte b[] = s.getBytes(utf8);
+      assert b.length == data.length;
+      System.arraycopy(b, 0, data, 0, b.length);
     }
 
     private byte[] generateRandomData(int n) {
@@ -503,7 +509,9 @@ public class TestPayloads extends Lucene
                 int freq = tp.freq();
                 for (int i = 0; i < freq; i++) {
                     tp.nextPosition();
-                    assertEquals(pool.bytesToString(tp.getPayload(new byte[5], 0)), terms.term().text);
+                    byte payload[] = new byte[5];
+                    tp.getPayload(payload, 0);
+                    assertEquals(terms.term().text, new String(payload, 0, payload.length, utf8));
                 }
             }
             tp.close();
@@ -527,7 +535,7 @@ public class TestPayloads extends Lucene
             this.pool = pool;
             payload = pool.get();
             generateRandomData(payload);
-            term = pool.bytesToString(payload);
+            term = new String(payload, 0, payload.length, utf8);
             first = true;
             payloadAtt = addAttribute(PayloadAttribute.class);
             termAtt = addAttribute(CharTermAttribute.class);
@@ -559,18 +567,6 @@ public class TestPayloads extends Lucene
                 pool.add(new byte[size]);
             }
         }
-        
-        private UnicodeUtil.UTF8Result utf8Result = new UnicodeUtil.UTF8Result();
-
-        synchronized String bytesToString(byte[] bytes) {
-            String s = new String(bytes);
-            UnicodeUtil.UTF16toUTF8(s, 0, s.length(), utf8Result);
-            try {
-                return new String(utf8Result.result, 0, utf8Result.length, "UTF-8");
-            } catch (UnsupportedEncodingException uee) {
-                return null;
-            }
-        }
     
         synchronized byte[] get() {
             return pool.remove(0);