svn commit: r942288 - in /lucene/dev/trunk/solr: CHANGES.txt src/java/org/apache/solr/core/SolrResourceLoader.java src/test/org/apache/solr/core/ResourceLoaderTest.java src/test/test-files/solr/conf/stopwithbom.txt

[rm...@apache.org]

Author: rmuir
Date: Sat May  8 00:42:04 2010
New Revision: 942288

URL: http://svn.apache.org/viewvc?rev=942288&view=rev
Log:
SOLR-1865: ignore BOMs in SolrResourceLoader.getLines

Added:
    lucene/dev/trunk/solr/src/test/test-files/solr/conf/stopwithbom.txt   (with props)
Modified:
    lucene/dev/trunk/solr/CHANGES.txt
    lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrResourceLoader.java
    lucene/dev/trunk/solr/src/test/org/apache/solr/core/ResourceLoaderTest.java

Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=942288&r1=942287&r2=942288&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Sat May  8 00:42:04 2010
@@ -347,6 +347,10 @@ Other Changes
   (Chris Male via rmuir)
 
 * SOLR-1851: luceneAutoCommit no longer has any effect - it has been remove (Mark Miller)
+
+* SOLR-1865: SolrResourceLoader.getLines ignores Byte Order Markers (BOMs) at the
+  beginning of input files, these are often created by editors such as Windows
+  Notepad.  (rmuir, hossman)
   
   
 Build

Modified: lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrResourceLoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrResourceLoader.java?rev=942288&r1=942287&r2=942288&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrResourceLoader.java (original)
+++ lucene/dev/trunk/solr/src/java/org/apache/solr/core/SolrResourceLoader.java Sat May  8 00:42:04 2010
@@ -309,6 +309,9 @@ public class SolrResourceLoader implemen
 
       lines = new ArrayList<String>();
       for (String word=null; (word=input.readLine())!=null;) {
+        // skip initial bom marker
+        if (lines.isEmpty() && word.length() > 0 && word.charAt(0) == '\uFEFF')
+          word = word.substring(1);
         // skip comments
         if (word.startsWith("#")) continue;
         word=word.trim();

Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/core/ResourceLoaderTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/core/ResourceLoaderTest.java?rev=942288&r1=942287&r2=942288&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/core/ResourceLoaderTest.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/core/ResourceLoaderTest.java Sat May  8 00:42:04 2010
@@ -30,6 +30,9 @@ import org.apache.solr.util.plugin.Resou
 import org.apache.solr.util.plugin.SolrCoreAware;
 
 import java.io.File;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
 
 public class ResourceLoaderTest extends TestCase 
 {
@@ -88,4 +91,30 @@ public class ResourceLoaderTest extends 
       catch( SolrException ex ) { } // OK
     }
   }
-}
\ No newline at end of file
+  
+  public void testBOMMarkers() throws Exception {
+    final String fileWithBom = "stopwithbom.txt";
+    SolrResourceLoader loader = new SolrResourceLoader(null);
+
+    // preliminary sanity check
+    InputStream bomStream = loader.openResource(fileWithBom);
+    try {
+      final byte[] bomExpected = new byte[] { -17, -69, -65 };
+      final byte[] firstBytes = new byte[3];
+      
+      assertEquals("Should have been able to read 3 bytes from bomStream",
+                   3, bomStream.read(firstBytes));
+
+      assertTrue("This test only works if " + fileWithBom + 
+                 " contains a BOM -- it appears someone removed it.", 
+                 Arrays.equals(bomExpected, firstBytes));
+    } finally {
+      try { bomStream.close(); } catch (Exception e) { /* IGNORE */ }
+    }
+
+    // now make sure getLines skips the BOM...
+    List<String> lines = loader.getLines(fileWithBom);
+    assertEquals(1, lines.size());
+    assertEquals("BOMsAreEvil", lines.get(0));
+  }
+}

Added: lucene/dev/trunk/solr/src/test/test-files/solr/conf/stopwithbom.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/test-files/solr/conf/stopwithbom.txt?rev=942288&view=auto
==============================================================================
--- lucene/dev/trunk/solr/src/test/test-files/solr/conf/stopwithbom.txt (added)
+++ lucene/dev/trunk/solr/src/test/test-files/solr/conf/stopwithbom.txt Sat May  8 00:42:04 2010
@@ -0,0 +1 @@
+BOMsAreEvil

Propchange: lucene/dev/trunk/solr/src/test/test-files/solr/conf/stopwithbom.txt
------------------------------------------------------------------------------
    svn:eol-style = native