Re: svn commit: r1308225 - in /lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal: ./ csv/ csv/writer/

[Robert Muir <rc...@gmail.com>]

Since i've already svn merge --reintegrated this branch, I don't think
we can safely merge any further changes... I think any future
development should be against trunk?

"In Subversion 1.5, once a --reintegrate merge is done from branch to
trunk, the branch is no longer usable for further work. It's not able
to correctly absorb new trunk changes, nor can it be properly
reintegrated to trunk again."

http://svnbook.red-bean.com/en/1.5/svn.branchmerge.basicmerging.html

On Sun, Apr 1, 2012 at 9:29 PM,  <ch...@apache.org> wrote:
> Author: chrism
> Date: Mon Apr  2 01:29:20 2012
> New Revision: 1308225
>
> URL: http://svn.apache.org/viewvc?rev=1308225&view=rev
> Log:
> LUCENE-3930: Imported commons-csv tests
>
> Added:
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
>    lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,581 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.internal.csv;
> +
> +import java.io.IOException;
> +import java.io.Reader;
> +import java.io.StringReader;
> +import java.util.Arrays;
> +
> +import junit.framework.TestCase;
> +
> +/**
> + * CSVParserTest
> + *
> + * The test are organized in three different sections:
> + * The 'setter/getter' section, the lexer section and finally the parser
> + * section. In case a test fails, you should follow a top-down approach for
> + * fixing a potential bug (its likely that the parser itself fails if the lexer
> + * has problems...).
> + */
> +public class CSVParserTest extends TestCase {
> +
> +  /**
> +   * TestCSVParser.
> +   */
> +  class TestCSVParser extends CSVParser {
> +    /**
> +     * Test parser to investigate the type of the internal Token.
> +     * @param in a Reader
> +     */
> +    TestCSVParser(Reader in) {
> +      super(in);
> +    }
> +
> +    TestCSVParser(Reader in, CSVStrategy strategy) {
> +      super(in, strategy);
> +    }
> +    /**
> +     * Calls super.nextToken() and prints out a String representation of token
> +     * type and content.
> +     * @return String representation of token type and content
> +     * @throws IOException like {@link CSVParser#nextToken()}
> +     */
> +    public String testNextToken() throws IOException {
> +      Token t = super.nextToken();
> +      return Integer.toString(t.type) + ";" + t.content + ";";
> +    }
> +  }
> +
> +  // ======================================================
> +  //   lexer tests
> +  // ======================================================
> +
> +  // Single line (without comment)
> +  public void testNextToken1() throws IOException {
> +    String code = "abc,def, hijk,  lmnop,   qrst,uv ,wxy   ,z , ,";
> +    TestCSVParser parser = new TestCSVParser(new StringReader(code));
> +    assertEquals(CSVParser.TT_TOKEN + ";abc;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";def;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";hijk;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";lmnop;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";qrst;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";uv;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";wxy;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";z;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> +  }
> +
> +  // multiline including comments (and empty lines)
> +  public void testNextToken2() throws IOException {
> +    /*   file:   1,2,3,
> +     *           a,b x,c
> +     *
> +     *           # this is a comment
> +     *           d,e,
> +     *
> +     */
> +    String code = "1,2,3,\na,b x,c\n#foo\n\nd,e,\n\n";
> +    CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> +    // strategy.setIgnoreEmptyLines(false);
> +    strategy.setCommentStart('#');
> +
> +    TestCSVParser parser = new TestCSVParser(new StringReader(code), strategy);
> +
> +
> +    assertEquals(CSVParser.TT_TOKEN + ";1;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";2;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";3;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";b x;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";c;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";d;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";e;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> +
> +  }
> +
> +  // simple token with escaping
> +  public void testNextToken3() throws IOException {
> +    /* file: a,\,,b
> +     *       \,,
> +     */
> +    String code = "a,\\,,b\n\\,,";
> +    CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> +    strategy.setCommentStart('#');
> +    TestCSVParser parser = new TestCSVParser(new StringReader(code), strategy);
> +
> +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +    // an unquoted single backslash is not an escape char
> +    assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> +    // an unquoted single backslash is not an escape char
> +    assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> +  }
> +
> +  // encapsulator tokenizer (sinle line)
> +  public void testNextToken4() throws IOException {
> +    /* file:  a,"foo",b
> +     *        a,   " foo",b
> +     *        a,"foo "   ,b     // whitespace after closing encapsulator
> +     *        a,  " foo " ,b
> +     */
> +     String code =
> +      "a,\"foo\",b\na,   \" foo\",b\na,\"foo \"  ,b\na,  \" foo \"  ,b";
> +     TestCSVParser parser = new TestCSVParser(new StringReader(code));
> +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + ";foo;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + "; foo;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + ";foo ;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_TOKEN + "; foo ;", parser.testNextToken());
> +//     assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> +     assertEquals(CSVParser.TT_EOF + ";b;", parser.testNextToken());
> +  }
> +
> +  // encapsulator tokenizer (multi line, delimiter in string)
> +  public void testNextToken5() throws IOException {
> +    String code =
> +      "a,\"foo\n\",b\n\"foo\n  baar ,,,\"\n\"\n\t \n\"";
> +    TestCSVParser parser = new TestCSVParser(new StringReader(code));
> +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_TOKEN + ";foo\n;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> +    assertEquals(CSVParser.TT_EORECORD + ";foo\n  baar ,,,;",
> +        parser.testNextToken());
> +    assertEquals(CSVParser.TT_EOF + ";\n\t \n;", parser.testNextToken());
> +
> +  }
> +
> +  // change delimiters, comment, encapsulater
> +  public void testNextToken6() throws IOException {
> +    /* file: a;'b and \' more
> +     *       '
> +     *       !comment;;;;
> +     *       ;;
> +     */
> +    String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
> +    TestCSVParser parser = new TestCSVParser(new StringReader(code), new CSVStrategy(';', '\'', '!'));
> +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> +    assertEquals(
> +      CSVParser.TT_EORECORD + ";b and ' more\n;",
> +      parser.testNextToken());
> +  }
> +
> +
> +  // ======================================================
> +  //   parser tests
> +  // ======================================================
> +
> +  String code =
> +    "a,b,c,d\n"
> +    + " a , b , 1 2 \n"
> +    + "\"foo baar\", b,\n"
> +   // + "   \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
> +      + "   \"foo\n,,\n\"\",,\n\"\"\",d,e\n";   // changed to use standard CSV escaping
> +  String[][] res = {
> +    {"a", "b", "c", "d"},
> +    {"a", "b", "1 2"},
> +    {"foo baar", "b", ""},
> +    {"foo\n,,\n\",,\n\"", "d", "e"}
> +  };
> +  public void testGetLine() throws IOException {
> +    CSVParser parser = new CSVParser(new StringReader(code));
> +    String[] tmp = null;
> +    for (int i = 0; i < res.length; i++) {
> +      tmp = parser.getLine();
> +      assertTrue(Arrays.equals(res[i], tmp));
> +    }
> +    tmp = parser.getLine();
> +    assertTrue(tmp == null);
> +  }
> +
> +  public void testNextValue() throws IOException {
> +    CSVParser parser = new CSVParser(new StringReader(code));
> +    String tmp = null;
> +    for (int i = 0; i < res.length; i++) {
> +      for (int j = 0; j < res[i].length; j++) {
> +        tmp = parser.nextValue();
> +        assertEquals(res[i][j], tmp);
> +      }
> +    }
> +    tmp = parser.nextValue();
> +    assertTrue(tmp == null);
> +  }
> +
> +  public void testGetAllValues() throws IOException {
> +    CSVParser parser = new CSVParser(new StringReader(code));
> +    String[][] tmp = parser.getAllValues();
> +    assertEquals(res.length, tmp.length);
> +    assertTrue(tmp.length > 0);
> +    for (int i = 0; i < res.length; i++) {
> +      assertTrue(Arrays.equals(res[i], tmp[i]));
> +    }
> +  }
> +
> +  public void testExcelStrategy1() throws IOException {
> +    String code =
> +      "value1,value2,value3,value4\r\na,b,c,d\r\n  x,,,"
> +      + "\r\n\r\n\"\"\"hello\"\"\",\"  \"\"world\"\"\",\"abc\ndef\",\r\n";
> +    String[][] res = {
> +      {"value1", "value2", "value3", "value4"},
> +      {"a", "b", "c", "d"},
> +      {"  x", "", "", ""},
> +      {""},
> +      {"\"hello\"", "  \"world\"", "abc\ndef", ""}
> +    };
> +    CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
> +    String[][] tmp = parser.getAllValues();
> +    assertEquals(res.length, tmp.length);
> +    assertTrue(tmp.length > 0);
> +    for (int i = 0; i < res.length; i++) {
> +      assertTrue(Arrays.equals(res[i], tmp[i]));
> +    }
> +  }
> +
> +  public void testExcelStrategy2() throws Exception {
> +    String code = "foo,baar\r\n\r\nhello,\r\n\r\nworld,\r\n";
> +    String[][] res = {
> +      {"foo", "baar"},
> +      {""},
> +      {"hello", ""},
> +      {""},
> +      {"world", ""}
> +    };
> +    CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
> +    String[][] tmp = parser.getAllValues();
> +    assertEquals(res.length, tmp.length);
> +    assertTrue(tmp.length > 0);
> +    for (int i = 0; i < res.length; i++) {
> +      assertTrue(Arrays.equals(res[i], tmp[i]));
> +    }
> +  }
> +
> +  public void testEndOfFileBehaviourExcel() throws Exception {
> +    String[] codes = {
> +        "hello,\r\n\r\nworld,\r\n",
> +        "hello,\r\n\r\nworld,",
> +        "hello,\r\n\r\nworld,\"\"\r\n",
> +        "hello,\r\n\r\nworld,\"\"",
> +        "hello,\r\n\r\nworld,\n",
> +        "hello,\r\n\r\nworld,",
> +        "hello,\r\n\r\nworld,\"\"\n",
> +        "hello,\r\n\r\nworld,\"\""
> +        };
> +    String[][] res = {
> +      {"hello", ""},
> +      {""},  // ExcelStrategy does not ignore empty lines
> +      {"world", ""}
> +    };
> +    String code;
> +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> +      code = codes[codeIndex];
> +      CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
> +      String[][] tmp = parser.getAllValues();
> +      assertEquals(res.length, tmp.length);
> +      assertTrue(tmp.length > 0);
> +      for (int i = 0; i < res.length; i++) {
> +        assertTrue(Arrays.equals(res[i], tmp[i]));
> +      }
> +    }
> +  }
> +
> +  public void testEndOfFileBehaviorCSV() throws Exception {
> +    String[] codes = {
> +        "hello,\r\n\r\nworld,\r\n",
> +        "hello,\r\n\r\nworld,",
> +        "hello,\r\n\r\nworld,\"\"\r\n",
> +        "hello,\r\n\r\nworld,\"\"",
> +        "hello,\r\n\r\nworld,\n",
> +        "hello,\r\n\r\nworld,",
> +        "hello,\r\n\r\nworld,\"\"\n",
> +        "hello,\r\n\r\nworld,\"\""
> +        };
> +    String[][] res = {
> +      {"hello", ""},  // CSV Strategy ignores empty lines
> +      {"world", ""}
> +    };
> +    String code;
> +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> +      code = codes[codeIndex];
> +      CSVParser parser = new CSVParser(new StringReader(code));
> +      String[][] tmp = parser.getAllValues();
> +      assertEquals(res.length, tmp.length);
> +      assertTrue(tmp.length > 0);
> +      for (int i = 0; i < res.length; i++) {
> +        assertTrue(Arrays.equals(res[i], tmp[i]));
> +      }
> +    }
> +  }
> +
> +  public void testEmptyLineBehaviourExcel() throws Exception {
> +    String[] codes = {
> +        "hello,\r\n\r\n\r\n",
> +        "hello,\n\n\n",
> +        "hello,\"\"\r\n\r\n\r\n",
> +        "hello,\"\"\n\n\n"
> +        };
> +    String[][] res = {
> +      {"hello", ""},
> +      {""},  // ExcelStrategy does not ignore empty lines
> +      {""}
> +    };
> +    String code;
> +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> +      code = codes[codeIndex];
> +      CSVParser parser = new CSVParser(new StringReader(code), CSVStrategy.EXCEL_STRATEGY);
> +      String[][] tmp = parser.getAllValues();
> +      assertEquals(res.length, tmp.length);
> +      assertTrue(tmp.length > 0);
> +      for (int i = 0; i < res.length; i++) {
> +        assertTrue(Arrays.equals(res[i], tmp[i]));
> +      }
> +    }
> +  }
> +
> +  public void testEmptyLineBehaviourCSV() throws Exception {
> +    String[] codes = {
> +        "hello,\r\n\r\n\r\n",
> +        "hello,\n\n\n",
> +        "hello,\"\"\r\n\r\n\r\n",
> +        "hello,\"\"\n\n\n"
> +        };
> +    String[][] res = {
> +      {"hello", ""}  // CSV Strategy ignores empty lines
> +    };
> +    String code;
> +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> +      code = codes[codeIndex];
> +      CSVParser parser = new CSVParser(new StringReader(code));
> +      String[][] tmp = parser.getAllValues();
> +      assertEquals(res.length, tmp.length);
> +      assertTrue(tmp.length > 0);
> +      for (int i = 0; i < res.length; i++) {
> +        assertTrue(Arrays.equals(res[i], tmp[i]));
> +      }
> +    }
> +  }
> +
> +  public void OLDtestBackslashEscaping() throws IOException {
> +    String code =
> +      "one,two,three\n"
> +      + "on\\\"e,two\n"
> +      + "on\"e,two\n"
> +      + "one,\"tw\\\"o\"\n"
> +      + "one,\"t\\,wo\"\n"
> +      + "one,two,\"th,ree\"\n"
> +      + "\"a\\\\\"\n"
> +      + "a\\,b\n"
> +      + "\"a\\\\,b\"";
> +    String[][] res = {
> +        { "one", "two", "three" },
> +        { "on\\\"e", "two" },
> +        { "on\"e", "two" },
> +        { "one", "tw\"o" },
> +        { "one", "t\\,wo" },  // backslash in quotes only escapes a delimiter (",")
> +        { "one", "two", "th,ree" },
> +        { "a\\\\" },     // backslash in quotes only escapes a delimiter (",")
> +        { "a\\", "b" },  // a backslash must be returnd
> +        { "a\\\\,b" }    // backslash in quotes only escapes a delimiter (",")
> +      };
> +    CSVParser parser = new CSVParser(new StringReader(code));
> +    String[][] tmp = parser.getAllValues();
> +    assertEquals(res.length, tmp.length);
> +    assertTrue(tmp.length > 0);
> +    for (int i = 0; i < res.length; i++) {
> +      assertTrue(Arrays.equals(res[i], tmp[i]));
> +    }
> +  }
> +
> +  public void testBackslashEscaping() throws IOException {
> +
> +    // To avoid confusion over the need for escaping chars in java code,
> +    // We will test with a forward slash as the escape char, and a single
> +    // quote as the encapsulator.
> +
> +    String code =
> +      "one,two,three\n" // 0
> +      + "'',''\n"       // 1) empty encapsulators
> +      + "/',/'\n"       // 2) single encapsulators
> +      + "'/'','/''\n"   // 3) single encapsulators encapsulated via escape
> +      + "'''',''''\n"   // 4) single encapsulators encapsulated via doubling
> +      + "/,,/,\n"       // 5) separator escaped
> +      + "//,//\n"       // 6) escape escaped
> +      + "'//','//'\n"   // 7) escape escaped in encapsulation
> +      + "   8   ,   \"quoted \"\" /\" // string\"   \n"     // don't eat spaces
> +      + "9,   /\n   \n"  // escaped newline
> +      + "";
> +    String[][] res = {
> +        { "one", "two", "three" }, // 0
> +        { "", "" },                // 1
> +        { "'", "'" },              // 2
> +        { "'", "'" },              // 3
> +        { "'", "'" },              // 4
> +        { ",", "," },              // 5
> +        { "/", "/" },              // 6
> +        { "/", "/" },              // 7
> +        { "   8   ", "   \"quoted \"\" \" / string\"   " },
> +        { "9", "   \n   " },
> +      };
> +
> +
> +    CSVStrategy strategy = new CSVStrategy(',','\'',CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true);
> +
> +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> +    String[][] tmp = parser.getAllValues();
> +    assertTrue(tmp.length > 0);
> +    for (int i = 0; i < res.length; i++) {
> +      assertTrue(Arrays.equals(res[i], tmp[i]));
> +    }
> +  }
> +
> +  public void testBackslashEscaping2() throws IOException {
> +
> +    // To avoid confusion over the need for escaping chars in java code,
> +    // We will test with a forward slash as the escape char, and a single
> +    // quote as the encapsulator.
> +
> +    String code = ""
> +      + " , , \n"           // 1)
> +      + " \t ,  , \n"       // 2)
> +      + " // , /, , /,\n"   // 3)
> +      + "";
> +    String[][] res = {
> +        { " ", " ", " " },         // 1
> +        { " \t ", "  ", " " },         // 2
> +        { " / ", " , ", " ," },         //3
> +      };
> +
> +
> +    CSVStrategy strategy = new CSVStrategy(',',CSVStrategy.ENCAPSULATOR_DISABLED,CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true);
> +
> +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> +    String[][] tmp = parser.getAllValues();
> +    assertTrue(tmp.length > 0);
> +
> +    if (!CSVPrinterTest.equals(res, tmp)) {
> +      assertTrue(false);
> +    }
> +
> +  }
> +
> +
> +  public void testDefaultStrategy() throws IOException {
> +
> +    String code = ""
> +        + "a,b\n"            // 1)
> +        + "\"\n\",\" \"\n"   // 2)
> +        + "\"\",#\n"   // 2)
> +        ;
> +    String[][] res = {
> +        { "a", "b" },
> +        { "\n", " " },
> +        { "", "#" },
> +    };
> +
> +    CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
> +    assertEquals(CSVStrategy.COMMENTS_DISABLED, strategy.getCommentStart());
> +
> +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> +    String[][] tmp = parser.getAllValues();
> +    assertTrue(tmp.length > 0);
> +
> +    if (!CSVPrinterTest.equals(res, tmp)) {
> +      assertTrue(false);
> +    }
> +
> +    String[][] res_comments = {
> +        { "a", "b" },
> +        { "\n", " " },
> +        { ""},
> +    };
> +
> +    strategy = new CSVStrategy(',','"','#');
> +    parser = new CSVParser(new StringReader(code), strategy);
> +    tmp = parser.getAllValues();
> +
> +    if (!CSVPrinterTest.equals(res_comments, tmp)) {
> +      assertTrue(false);
> +    }
> +  }
> +
> +
> +    public void testUnicodeEscape() throws IOException {
> +      String code = "abc,\\u0070\\u0075\\u0062\\u006C\\u0069\\u0063";
> +      CSVParser parser = new CSVParser(new StringReader(code));
> +      parser.getStrategy().setUnicodeEscapeInterpretation(true);
> +      String[] data = parser.getLine();
> +      assertEquals(2, data.length);
> +      assertEquals("abc", data[0]);
> +      assertEquals("public", data[1]);
> +    }
> +
> +    public void testCarriageReturnLineFeedEndings() throws IOException {
> +     String code = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
> +     CSVParser parser = new CSVParser(new StringReader(code));
> +     String[][] data = parser.getAllValues();
> +     assertEquals(4, data.length);
> +    }
> +
> +    public void testIgnoreEmptyLines() throws IOException {
> +      String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
> +      //String code = "world\r\n\n";
> +      //String code = "foo;baar\r\n\r\nhello;\r\n\r\nworld;\r\n";
> +      CSVParser parser = new CSVParser(new StringReader(code));
> +      String[][] data = parser.getAllValues();
> +      assertEquals(3, data.length);
> +    }
> +
> +    public void testLineTokenConsistency() throws IOException {
> +      String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
> +      CSVParser parser = new CSVParser(new StringReader(code));
> +      String[][] data = parser.getAllValues();
> +      parser = new CSVParser(new StringReader(code));
> +      CSVParser parser1 = new CSVParser(new StringReader(code));
> +      for (int i = 0; i < data.length; i++) {
> +        assertTrue(Arrays.equals(parser1.getLine(), data[i]));
> +        for (int j = 0; j < data[i].length; j++) {
> +          assertEquals(parser.nextValue(), data[i][j]);
> +        }
> +      }
> +    }
> +
> +    // From SANDBOX-153
> +     public void testDelimiterIsWhitespace() throws IOException {
> +         String code = "one\ttwo\t\tfour \t five\t six";
> +         TestCSVParser parser = new TestCSVParser(new StringReader(code), CSVStrategy.TDF_STRATEGY);
> +         assertEquals(CSVParser.TT_TOKEN + ";one;", parser.testNextToken());
> +         assertEquals(CSVParser.TT_TOKEN + ";two;", parser.testNextToken());
> +         assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> +         assertEquals(CSVParser.TT_TOKEN + ";four;", parser.testNextToken());
> +         assertEquals(CSVParser.TT_TOKEN + ";five;", parser.testNextToken());
> +         assertEquals(CSVParser.TT_EOF + ";six;", parser.testNextToken());
> +     }
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,197 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.internal.csv;
> +
> +import java.io.IOException;
> +import java.io.StringReader;
> +import java.io.StringWriter;
> +import java.util.Arrays;
> +import java.util.Random;
> +
> +import junit.framework.Test;
> +import junit.framework.TestCase;
> +import junit.framework.TestSuite;
> +
> +/**
> + * CSVPrinterTest
> + */
> +public class CSVPrinterTest extends TestCase {
> +
> +  String lineSeparator = "\n";
> +
> +  public void testPrinter1() throws IOException {
> +    StringWriter sw = new StringWriter();
> +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.DEFAULT_STRATEGY);
> +    String[] line1 = {"a", "b"};
> +    printer.println(line1);
> +    assertEquals("a,b" + lineSeparator, sw.toString());
> +  }
> +
> +  public void testPrinter2() throws IOException {
> +    StringWriter sw = new StringWriter();
> +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.DEFAULT_STRATEGY);
> +    String[] line1 = {"a,b", "b"};
> +    printer.println(line1);
> +    assertEquals("\"a,b\",b" + lineSeparator, sw.toString());
> +  }
> +
> +  public void testPrinter3() throws IOException {
> +    StringWriter sw = new StringWriter();
> +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.DEFAULT_STRATEGY);
> +    String[] line1 = {"a, b", "b "};
> +    printer.println(line1);
> +    assertEquals("\"a, b\",\"b \"" + lineSeparator, sw.toString());
> +  }
> +
> +  public void testExcelPrinter1() throws IOException {
> +    StringWriter sw = new StringWriter();
> +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.EXCEL_STRATEGY);
> +    String[] line1 = {"a", "b"};
> +    printer.println(line1);
> +    assertEquals("a,b" + lineSeparator, sw.toString());
> +  }
> +
> +  public void testExcelPrinter2() throws IOException {
> +    StringWriter sw = new StringWriter();
> +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.EXCEL_STRATEGY);
> +    String[] line1 = {"a,b", "b"};
> +    printer.println(line1);
> +    assertEquals("\"a,b\",b" + lineSeparator, sw.toString());
> +  }
> +
> +
> +
> +  public void testRandom() throws Exception {
> +    int iter=10000;
> +    strategy = CSVStrategy.DEFAULT_STRATEGY;
> +    doRandom(iter);
> +    strategy = CSVStrategy.EXCEL_STRATEGY;
> +    doRandom(iter);
> +
> +    // Strategy for MySQL
> +    strategy = new CSVStrategy('\t', CSVStrategy.ENCAPSULATOR_DISABLED, CSVStrategy.COMMENTS_DISABLED,'\\',false, false, false, false);
> +    doRandom(iter);
> +  }
> +
> +  Random r = new Random();
> +  CSVStrategy strategy;
> +
> +  public void doRandom(int iter) throws Exception {
> +    for (int i=0; i<iter; i++) {
> +      doOneRandom();
> +    }
> +  }
> +
> +  public void doOneRandom() throws Exception {
> +    int nLines = r.nextInt(4)+1;
> +    int nCol = r.nextInt(3)+1;
> +    // nLines=1;nCol=2;
> +    String[][] lines = new String[nLines][];
> +    for (int i=0; i<nLines; i++) {
> +      String[] line = new String[nCol];
> +      lines[i] = line;
> +      for (int j=0; j<nCol; j++) {
> +        line[j] = randStr();
> +      }
> +    }
> +
> +    StringWriter sw = new StringWriter();
> +    CSVPrinter printer = new CSVPrinter(sw, strategy);
> +
> +    for (int i=0; i<nLines; i++) {
> +      // for (int j=0; j<lines[i].length; j++) System.out.println("### VALUE=:" + printable(lines[i][j]));
> +      printer.println(lines[i]);
> +    }
> +
> +    printer.flush();
> +    String result = sw.toString();
> +    // System.out.println("### :" + printable(result));
> +
> +    StringReader reader = new StringReader(result);
> +
> +    CSVParser parser = new CSVParser(reader, strategy);
> +    String[][] parseResult = parser.getAllValues();
> +
> +    if (!equals(lines, parseResult)) {
> +      System.out.println("Printer output :" + printable(result));
> +      assertTrue(false);
> +    }
> +  }
> +
> +  public static boolean equals(String[][] a, String[][] b) {
> +    if (a.length != b.length) {
> +      return false;
> +    }
> +    for (int i=0; i<a.length; i++) {
> +      String[] linea = a[i];
> +      String[] lineb = b[i];
> +      if (linea.length != lineb.length) {
> +        return false;
> +      }
> +      for (int j=0; j<linea.length; j++) {
> +        String aval = linea[j];
> +        String bval = lineb[j];
> +        if (!aval.equals(bval)) {
> +          System.out.println("expected  :" + printable(aval));
> +          System.out.println("got       :" + printable(bval));
> +          return false;
> +        }
> +      }
> +    }
> +    return true;
> +  }
> +
> +  public static String printable(String s) {
> +    StringBuffer sb = new StringBuffer();
> +    for (int i=0; i<s.length(); i++) {
> +      char ch = s.charAt(i);
> +      if (ch<=' ' || ch>=128) {
> +        sb.append("(" + (int)ch + ")");
> +      } else {
> +        sb.append(ch);
> +      }
> +    }
> +    return sb.toString();
> +  }
> +
> +  public String randStr() {
> +    int sz = r.nextInt(20);
> +    // sz = r.nextInt(3);
> +    char[] buf = new char[sz];
> +    for (int i=0; i<sz; i++) {
> +      // stick in special chars with greater frequency
> +      char ch;
> +      int what = r.nextInt(20);
> +      switch (what) {
> +        case 0: ch = '\r'; break;
> +        case 1: ch = '\n'; break;
> +        case 2: ch = '\t'; break;
> +        case 3: ch = '\f'; break;
> +        case 4: ch = ' ';  break;
> +        case 5: ch = ',';  break;
> +        case 6: ch = '"';  break;
> +        case 7: ch = '\''; break;
> +        case 8: ch = '\\'; break;
> +        default: ch = (char)r.nextInt(300); break;
> +        // default: ch = 'a'; break;
> +      }
> +      buf[i] = ch;
> +    }
> +    return new String(buf);
> +  }
> +
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,91 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.internal.csv;
> +
> +import java.io.StringReader;
> +
> +import junit.framework.TestCase;
> +
> +/**
> + * CSVStrategyTest
> + *
> + * The test are organized in three different sections:
> + * The 'setter/getter' section, the lexer section and finally the strategy
> + * section. In case a test fails, you should follow a top-down approach for
> + * fixing a potential bug (its likely that the strategy itself fails if the lexer
> + * has problems...).
> + */
> +public class CSVStrategyTest extends TestCase {
> +
> +  // ======================================================
> +  //   getters / setters
> +  // ======================================================
> +  public void testGetSetCommentStart() {
> +    CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> +    strategy.setCommentStart('#');
> +    assertEquals(strategy.getCommentStart(), '#');
> +    strategy.setCommentStart('!');
> +    assertEquals(strategy.getCommentStart(), '!');
> +  }
> +
> +  public void testGetSetEncapsulator() {
> +    CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> +    strategy.setEncapsulator('"');
> +    assertEquals(strategy.getEncapsulator(), '"');
> +    strategy.setEncapsulator('\'');
> +    assertEquals(strategy.getEncapsulator(), '\'');
> +  }
> +
> +  public void testGetSetDelimiter() {
> +    CSVStrategy strategy = (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> +    strategy.setDelimiter(';');
> +    assertEquals(strategy.getDelimiter(), ';');
> +    strategy.setDelimiter(',');
> +    assertEquals(strategy.getDelimiter(), ',');
> +    strategy.setDelimiter('\t');
> +    assertEquals(strategy.getDelimiter(), '\t');
> +  }
> +
> +  public void testSetCSVStrategy() {
> +    CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
> +    // default settings
> +    assertEquals(strategy.getDelimiter(), ',');
> +    assertEquals(strategy.getEncapsulator(), '"');
> +    assertEquals(strategy.getCommentStart(), CSVStrategy.COMMENTS_DISABLED);
> +    assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
> +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> +    assertEquals(true,  strategy.getIgnoreEmptyLines());
> +    // explicit csv settings
> +    assertEquals(strategy.getDelimiter(), ',');
> +    assertEquals(strategy.getEncapsulator(), '"');
> +    assertEquals(strategy.getCommentStart(), CSVStrategy.COMMENTS_DISABLED);
> +    assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
> +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> +    assertEquals(true,  strategy.getIgnoreEmptyLines());
> +  }
> +
> +  public void testSetExcelStrategy() {
> +    CSVStrategy strategy = CSVStrategy.EXCEL_STRATEGY;
> +    assertEquals(strategy.getDelimiter(), ',');
> +    assertEquals(strategy.getEncapsulator(), '"');
> +    assertEquals(strategy.getCommentStart(), CSVStrategy.COMMENTS_DISABLED);
> +    assertEquals(false,  strategy.getIgnoreLeadingWhitespaces());
> +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> +    assertEquals(false, strategy.getIgnoreEmptyLines());
> +  }
> +
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,150 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.internal.csv;
> +
> +import java.io.IOException;
> +
> +import junit.framework.TestCase;
> +
> +/**
> + * CSVUtilsTest
> + */
> +public class CSVUtilsTest extends TestCase {
> +
> +  // ======================================================
> +  //   static parser tests
> +  // ======================================================
> +  public void testParse1() throws IOException {
> +      String[][] data = CSVUtils.parse("abc\ndef");
> +      assertEquals(2, data.length);
> +      assertEquals(1, data[0].length);
> +      assertEquals(1, data[1].length);
> +      assertEquals("abc", data[0][0]);
> +      assertEquals("def", data[1][0]);
> +    }
> +
> +    public void testParse2() throws IOException {
> +      String[][] data = CSVUtils.parse("abc,def,\"ghi,jkl\"\ndef");
> +      assertEquals(2, data.length);
> +      assertEquals(3, data[0].length);
> +      assertEquals(1, data[1].length);
> +      assertEquals("abc", data[0][0]);
> +      assertEquals("def", data[0][1]);
> +      assertEquals("ghi,jkl", data[0][2]);
> +      assertEquals("def", data[1][0]);
> +    }
> +
> +    public void testParse3() throws IOException {
> +      String[][] data = CSVUtils.parse("abc,\"def\nghi\"\njkl");
> +      assertEquals(2, data.length);
> +      assertEquals(2, data[0].length);
> +      assertEquals(1, data[1].length);
> +      assertEquals("abc", data[0][0]);
> +      assertEquals("def\nghi", data[0][1]);
> +      assertEquals("jkl", data[1][0]);
> +    }
> +
> +    public void testParse4() throws IOException {
> +      String[][] data = CSVUtils.parse("abc,\"def\\\\nghi\"\njkl");
> +      assertEquals(2, data.length);
> +      assertEquals(2, data[0].length);
> +      assertEquals(1, data[1].length);
> +      assertEquals("abc", data[0][0]);
> +      // an escape char in quotes only escapes a delimiter, not itself
> +      assertEquals("def\\\\nghi", data[0][1]);
> +      assertEquals("jkl", data[1][0]);
> +    }
> +
> +    public void testParse5() throws IOException {
> +      String[][] data = CSVUtils.parse("abc,def\\nghi\njkl");
> +      assertEquals(2, data.length);
> +      assertEquals(2, data[0].length);
> +      assertEquals(1, data[1].length);
> +      assertEquals("abc", data[0][0]);
> +      assertEquals("def\\nghi", data[0][1]);
> +      assertEquals("jkl", data[1][0]);
> +    }
> +
> +    public void testParse6() throws IOException {
> +      String[][] data = CSVUtils.parse("");
> +      // default strategy is CSV, which ignores empty lines
> +      assertEquals(0, data.length);
> +    }
> +
> +    public void testParse7() throws IOException {
> +      boolean io = false;
> +      try {
> +        CSVUtils.parse(null);
> +      } catch (IllegalArgumentException e) {
> +        io = true;
> +      }
> +      assertTrue(io);
> +    }
> +
> +    public void testParseLine1() throws IOException {
> +      String[] data = CSVUtils.parseLine("abc,def,ghi");
> +      assertEquals(3, data.length);
> +      assertEquals("abc", data[0]);
> +      assertEquals("def", data[1]);
> +      assertEquals("ghi", data[2]);
> +    }
> +
> +    public void testParseLine2() throws IOException {
> +      String[] data = CSVUtils.parseLine("abc,def,ghi\n");
> +      assertEquals(3, data.length);
> +      assertEquals("abc", data[0]);
> +      assertEquals("def", data[1]);
> +      assertEquals("ghi", data[2]);
> +    }
> +
> +    public void testParseLine3() throws IOException {
> +      String[] data = CSVUtils.parseLine("abc,\"def,ghi\"");
> +      assertEquals(2, data.length);
> +      assertEquals("abc", data[0]);
> +      assertEquals("def,ghi", data[1]);
> +    }
> +
> +    public void testParseLine4() throws IOException {
> +      String[] data = CSVUtils.parseLine("abc,\"def\nghi\"");
> +      assertEquals(2, data.length);
> +      assertEquals("abc", data[0]);
> +      assertEquals("def\nghi", data[1]);
> +    }
> +
> +    public void testParseLine5() throws IOException {
> +      String[] data = CSVUtils.parseLine("");
> +      assertEquals(0, data.length);
> +      // assertEquals("", data[0]);
> +    }
> +
> +    public void testParseLine6() throws IOException {
> +      boolean io = false;
> +      try {
> +        CSVUtils.parseLine(null);
> +      } catch (IllegalArgumentException e) {
> +        io = true;
> +      }
> +      assertTrue(io);
> +    }
> +
> +    public void testParseLine7() throws IOException {
> +      String[] res = CSVUtils.parseLine("");
> +      assertNotNull(res);
> +      assertEquals(0, res.length);
> +    }
> +
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,192 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.solr.internal.csv;
> +
> +import junit.framework.TestCase;
> +
> +public class CharBufferTest extends TestCase {
> +    public void testCreate() {
> +        CharBuffer cb = new CharBuffer();
> +        assertEquals(0, cb.length());
> +        try {
> +            cb = new CharBuffer(0);
> +            fail("Should not be possible");
> +        } catch(IllegalArgumentException e) {
> +            // expected
> +        }
> +
> +        cb = new CharBuffer(128);
> +        assertEquals(0, cb.length());
> +    }
> +
> +    public void testAppendChar() {
> +        CharBuffer cb = new CharBuffer(1);
> +        String expected = "";
> +        for (char c = 'a'; c < 'z'; c++) {
> +            cb.append(c);
> +            expected += c;
> +            assertEquals(expected, cb.toString());
> +            assertEquals(expected.length(), cb.length());
> +        }
> +    }
> +
> +    public void testAppendCharArray() {
> +        CharBuffer cb = new CharBuffer(1);
> +        char[] abcd = "abcd".toCharArray();
> +        String expected = "";
> +        for (int i=0; i<10; i++) {
> +            cb.append(abcd);
> +            expected += "abcd";
> +            assertEquals(expected, cb.toString());
> +            assertEquals(4*(i+1), cb.length());
> +        }
> +    }
> +
> +    public void testAppendString() {
> +        CharBuffer cb = new CharBuffer(1);
> +        String abcd = "abcd";
> +        String expected = "";
> +        for (int i=0; i<10; i++) {
> +            cb.append(abcd);
> +            expected += abcd;
> +            assertEquals(expected, cb.toString());
> +            assertEquals(4*(i+1), cb.length());
> +        }
> +    }
> +
> +    public void testAppendStringBuffer() {
> +        CharBuffer cb = new CharBuffer(1);
> +        StringBuffer abcd = new StringBuffer("abcd");
> +        String expected = "";
> +        for (int i=0; i<10; i++) {
> +            cb.append(abcd);
> +            expected += "abcd";
> +            assertEquals(expected, cb.toString());
> +            assertEquals(4*(i+1), cb.length());
> +        }
> +    }
> +
> +    public void testAppendCharBuffer() {
> +        CharBuffer cb = new CharBuffer(1);
> +        CharBuffer abcd = new CharBuffer(17);
> +        abcd.append("abcd");
> +        String expected = "";
> +        for (int i=0; i<10; i++) {
> +            cb.append(abcd);
> +            expected += "abcd";
> +            assertEquals(expected, cb.toString());
> +            assertEquals(4*(i+1), cb.length());
> +        }
> +    }
> +
> +    public void testShrink() {
> +        String data = "123456789012345678901234567890";
> +
> +        CharBuffer cb = new CharBuffer(data.length() + 100);
> +        assertEquals(data.length() + 100, cb.capacity());
> +        cb.append(data);
> +        assertEquals(data.length() + 100, cb.capacity());
> +        assertEquals(data.length(), cb.length());
> +        cb.shrink();
> +        assertEquals(data.length(), cb.capacity());
> +        assertEquals(data.length(), cb.length());
> +        assertEquals(data, cb.toString());
> +    }
> +
> +    //-- the following test cases have been adapted from the HttpComponents project
> +    //-- written by Oleg Kalnichevski
> +
> +    public void testSimpleAppend() throws Exception {
> +        CharBuffer buffer = new CharBuffer(16);
> +        assertEquals(16, buffer.capacity());
> +        assertEquals(0, buffer.length());
> +        char[] b1 = buffer.getCharacters();
> +        assertNotNull(b1);
> +        assertEquals(0, b1.length);
> +        assertEquals(0, buffer.length());
> +
> +        char[] tmp = new char[] { '1', '2', '3', '4'};
> +        buffer.append(tmp);
> +        assertEquals(16, buffer.capacity());
> +        assertEquals(4, buffer.length());
> +
> +        char[] b2 = buffer.getCharacters();
> +        assertNotNull(b2);
> +        assertEquals(4, b2.length);
> +        for (int i = 0; i < tmp.length; i++) {
> +            assertEquals(tmp[i], b2[i]);
> +        }
> +        assertEquals("1234", buffer.toString());
> +
> +        buffer.clear();
> +        assertEquals(16, buffer.capacity());
> +        assertEquals(0, buffer.length());
> +    }
> +
> +    public void testAppendString2() throws Exception {
> +        CharBuffer buffer = new CharBuffer(8);
> +        buffer.append("stuff");
> +        buffer.append(" and more stuff");
> +        assertEquals("stuff and more stuff", buffer.toString());
> +    }
> +
> +    public void testAppendNull() throws Exception {
> +        CharBuffer buffer = new CharBuffer(8);
> +
> +        buffer.append((StringBuffer)null);
> +        assertEquals("", buffer.toString());
> +
> +        buffer.append((String)null);
> +        assertEquals("", buffer.toString());
> +
> +        buffer.append((CharBuffer)null);
> +        assertEquals("", buffer.toString());
> +
> +        buffer.append((char[])null);
> +        assertEquals("", buffer.toString());
> +    }
> +
> +    public void testAppendCharArrayBuffer() throws Exception {
> +        CharBuffer buffer1 = new CharBuffer(8);
> +        buffer1.append(" and more stuff");
> +        CharBuffer buffer2 = new CharBuffer(8);
> +        buffer2.append("stuff");
> +        buffer2.append(buffer1);
> +        assertEquals("stuff and more stuff", buffer2.toString());
> +    }
> +
> +    public void testAppendSingleChar() throws Exception {
> +        CharBuffer buffer = new CharBuffer(4);
> +        buffer.append('1');
> +        buffer.append('2');
> +        buffer.append('3');
> +        buffer.append('4');
> +        buffer.append('5');
> +        buffer.append('6');
> +        assertEquals("123456", buffer.toString());
> +    }
> +
> +    public void testProvideCapacity() throws Exception {
> +        CharBuffer buffer = new CharBuffer(4);
> +        buffer.provideCapacity(2);
> +        assertEquals(4, buffer.capacity());
> +        buffer.provideCapacity(8);
> +        assertTrue(buffer.capacity() >= 8);
> +    }
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,222 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one or more
> + * contributor license agreements.  See the NOTICE file distributed with
> + * this work for additional information regarding copyright ownership.
> + * The ASF licenses this file to You under the Apache License, Version 2.0
> + * (the "License"); you may not use this file except in compliance with
> + * the License.  You may obtain a copy of the License at
> + *
> + *      http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.solr.internal.csv;
> +
> +import java.io.StringReader;
> +import java.util.Arrays;
> +
> +import junit.framework.Test;
> +import junit.framework.TestCase;
> +import junit.framework.TestSuite;
> +
> +/**
> + * ExtendedBufferedReaderTest
> + *
> + */
> +public class ExtendedBufferedReaderTest extends TestCase {
> +
> +  // ======================================================
> +  //   the test cases
> +  // ======================================================
> +
> +  public void testConstructors() {
> +    ExtendedBufferedReader br = new ExtendedBufferedReader(new StringReader(""));
> +    br = new ExtendedBufferedReader(new StringReader(""), 10);
> +  }
> +
> +  public void testReadLookahead1() throws Exception {
> +
> +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, getEBR("").read());
> +    ExtendedBufferedReader br = getEBR("1\n2\r3\n");
> +    assertEquals('1', br.lookAhead());
> +    assertEquals(ExtendedBufferedReader.UNDEFINED, br.readAgain());
> +    assertEquals('1', br.read());
> +    assertEquals('1', br.readAgain());
> +
> +    assertEquals(0, br.getLineNumber());
> +    assertEquals('\n', br.lookAhead());
> +    assertEquals(0, br.getLineNumber());
> +    assertEquals('1', br.readAgain());
> +    assertEquals('\n', br.read());
> +    assertEquals(1, br.getLineNumber());
> +    assertEquals('\n', br.readAgain());
> +    assertEquals(1, br.getLineNumber());
> +
> +    assertEquals('2', br.lookAhead());
> +    assertEquals(1, br.getLineNumber());
> +    assertEquals('\n', br.readAgain());
> +    assertEquals(1, br.getLineNumber());
> +    assertEquals('2', br.read());
> +    assertEquals('2', br.readAgain());
> +
> +    assertEquals('\r', br.lookAhead());
> +    assertEquals('2', br.readAgain());
> +    assertEquals('\r', br.read());
> +    assertEquals('\r', br.readAgain());
> +
> +    assertEquals('3', br.lookAhead());
> +    assertEquals('\r', br.readAgain());
> +    assertEquals('3', br.read());
> +    assertEquals('3', br.readAgain());
> +
> +    assertEquals('\n', br.lookAhead());
> +    assertEquals(1, br.getLineNumber());
> +    assertEquals('3', br.readAgain());
> +    assertEquals('\n', br.read());
> +    assertEquals(2, br.getLineNumber());
> +    assertEquals('\n', br.readAgain());
> +    assertEquals(2, br.getLineNumber());
> +
> +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.lookAhead());
> +    assertEquals('\n', br.readAgain());
> +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.read());
> +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.readAgain());
> +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.read());
> +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.lookAhead());
> +
> +  }
> +
> +
> +  public void testReadLookahead2() throws Exception {
> +    char[] ref = new char[5];
> +    char[] res = new char[5];
> +
> +    ExtendedBufferedReader br = getEBR("");
> +    assertEquals(0, br.read(res, 0, 0));
> +    assertTrue(Arrays.equals(res, ref));
> +
> +    br = getEBR("abcdefg");
> +    ref[0] = 'a';
> +    ref[1] = 'b';
> +    ref[2] = 'c';
> +    assertEquals(3, br.read(res, 0, 3));
> +    assertTrue(Arrays.equals(res, ref));
> +    assertEquals('c', br.readAgain());
> +
> +    assertEquals('d', br.lookAhead());
> +    ref[4] = 'd';
> +    assertEquals(1, br.read(res, 4, 1));
> +    assertTrue(Arrays.equals(res, ref));
> +    assertEquals('d', br.readAgain());
> +
> +  }
> +
> +  public void testMarkSupported() {
> +    assertFalse(getEBR("foo").markSupported());
> +  }
> +
> +  public void testReadLine() throws Exception {
> +    ExtendedBufferedReader br = getEBR("");
> +    assertTrue(br.readLine() == null);
> +
> +    br = getEBR("\n");
> +    assertTrue(br.readLine().equals(""));
> +    assertTrue(br.readLine() == null);
> +
> +    br = getEBR("foo\n\nhello");
> +    assertEquals(0, br.getLineNumber());
> +    assertTrue(br.readLine().equals("foo"));
> +    assertEquals(1, br.getLineNumber());
> +    assertTrue(br.readLine().equals(""));
> +    assertEquals(2, br.getLineNumber());
> +    assertTrue(br.readLine().equals("hello"));
> +    assertEquals(3, br.getLineNumber());
> +    assertTrue(br.readLine() == null);
> +    assertEquals(3, br.getLineNumber());
> +
> +    br = getEBR("foo\n\nhello");
> +    assertEquals('f', br.read());
> +    assertEquals('o', br.lookAhead());
> +    assertTrue(br.readLine().equals("oo"));
> +    assertEquals(1, br.getLineNumber());
> +    assertEquals('\n', br.lookAhead());
> +    assertTrue(br.readLine().equals(""));
> +    assertEquals(2, br.getLineNumber());
> +    assertEquals('h', br.lookAhead());
> +    assertTrue(br.readLine().equals("hello"));
> +    assertTrue(br.readLine() == null);
> +    assertEquals(3, br.getLineNumber());
> +
> +
> +    br = getEBR("foo\rbaar\r\nfoo");
> +    assertTrue(br.readLine().equals("foo"));
> +    assertEquals('b', br.lookAhead());
> +    assertTrue(br.readLine().equals("baar"));
> +    assertEquals('f', br.lookAhead());
> +    assertTrue(br.readLine().equals("foo"));
> +    assertTrue(br.readLine() == null);
> +  }
> +
> +  public void testSkip0() throws Exception {
> +
> +    ExtendedBufferedReader br = getEBR("");
> +    assertEquals(0, br.skip(0));
> +    assertEquals(0, br.skip(1));
> +
> +    br = getEBR("");
> +    assertEquals(0, br.skip(1));
> +
> +    br = getEBR("abcdefg");
> +    assertEquals(0, br.skip(0));
> +    assertEquals('a', br.lookAhead());
> +    assertEquals(0, br.skip(0));
> +    assertEquals('a', br.lookAhead());
> +    assertEquals(1, br.skip(1));
> +    assertEquals('b', br.lookAhead());
> +    assertEquals('b', br.read());
> +    assertEquals(3, br.skip(3));
> +    assertEquals('f', br.lookAhead());
> +    assertEquals(2, br.skip(5));
> +    assertTrue(br.readLine() == null);
> +
> +    br = getEBR("12345");
> +    assertEquals(5, br.skip(5));
> +    assertTrue (br.lookAhead() == ExtendedBufferedReader.END_OF_STREAM);
> +  }
> +
> +  public void testSkipUntil() throws Exception {
> +    ExtendedBufferedReader br = getEBR("");
> +    assertEquals(0, br.skipUntil(';'));
> +    br = getEBR("ABCDEF,GHL,,MN");
> +    assertEquals(6, br.skipUntil(','));
> +    assertEquals(0, br.skipUntil(','));
> +    br.skip(1);
> +    assertEquals(3, br.skipUntil(','));
> +    br.skip(1);
> +    assertEquals(0, br.skipUntil(','));
> +    br.skip(1);
> +    assertEquals(2, br.skipUntil(','));
> +  }
> +
> +  public void testReadUntil() throws Exception {
> +    ExtendedBufferedReader br = getEBR("");
> +    assertTrue(br.readUntil(';').equals(""));
> +    br = getEBR("ABCDEF;GHL;;MN");
> +    assertTrue(br.readUntil(';').equals("ABCDEF"));
> +    assertTrue(br.readUntil(';').length() == 0);
> +    br.skip(1);
> +    assertTrue(br.readUntil(';').equals("GHL"));
> +    br.skip(1);
> +    assertTrue(br.readUntil(';').equals(""));
> +    br.skip(1);
> +    assertTrue(br.readUntil(',').equals("MN"));
> +  }
> +
> +  private ExtendedBufferedReader getEBR(String s) {
> +    return new ExtendedBufferedReader(new StringReader(s));
> +  }
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,88 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.solr.internal.csv.writer;
> +
> +import java.io.ByteArrayInputStream;
> +
> +import junit.framework.TestCase;
> +
> +/**
> + * Tests for the config guesser.
> + */
> +public class CSVConfigGuesserTest extends TestCase {
> +
> +    public void testSetters() throws Exception {
> +        CSVConfigGuesser guesser = new CSVConfigGuesser();
> +        ByteArrayInputStream in = new ByteArrayInputStream(new byte[0]);
> +        guesser.setInputStream(in);
> +        assertEquals(in, guesser.getInputStream());
> +        guesser = new CSVConfigGuesser(in);
> +        assertEquals(in, guesser.getInputStream());
> +        assertEquals(false, guesser.hasFieldHeader());
> +        guesser.setHasFieldHeader(true);
> +        assertEquals(true, guesser.hasFieldHeader());
> +    }
> +    /**
> +     * Test a format like
> +     *  1234 ; abcd ; 1234 ;
> +     *
> +     */
> +    public void testConfigGuess1() {
> +        CSVConfig expected = new CSVConfig();
> +        expected.setDelimiter(';');
> +        expected.setValueDelimiter(' ');
> +        expected.setFill(CSVConfig.FILLRIGHT);
> +        expected.setIgnoreValueDelimiter(false);
> +        expected.setFixedWidth(true);
> +        CSVField field = new CSVField();
> +        field.setSize(4);
> +        expected.addField(field);
> +        expected.addField(field);
> +        StringBuffer sb = new StringBuffer();
> +        sb.append("1234;abcd;1234\n");
> +        sb.append("abcd;1234;abcd");
> +        ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
> +        CSVConfigGuesser guesser = new CSVConfigGuesser(in);
> +        CSVConfig guessed = guesser.guess();
> +        assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
> +        assertEquals(expected.getFields().length, guessed.getFields().length);
> +        assertEquals(expected.getFields()[0].getSize(), guessed.getFields()[0].getSize());
> +    }
> +    /**
> +     * Test a format like
> +     *  1234,123123,12312312,213123
> +     *  1,2,3,4
> +     *
> +     */
> +    public void testConfigGuess2() {
> +        CSVConfig expected = new CSVConfig();
> +        expected.setDelimiter(';');
> +        expected.setValueDelimiter(' ');
> +        expected.setFill(CSVConfig.FILLRIGHT);
> +        expected.setIgnoreValueDelimiter(false);
> +//        expected.setFixedWidth(false);
> +        StringBuffer sb = new StringBuffer();
> +        sb.append("1,2,3,4\n");
> +        sb.append("abcd,1234,abcd,1234");
> +        ByteArrayInputStream in = new ByteArrayInputStream(sb.toString().getBytes());
> +        CSVConfigGuesser guesser = new CSVConfigGuesser(in);
> +        CSVConfig guessed = guesser.guess();
> +        assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
> +    }
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,100 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.solr.internal.csv.writer;
> +
> +import java.util.Collection;
> +
> +import junit.framework.TestCase;
> +
> +/**
> + * Testcase for the CSVConfig
> + */
> +public class CSVConfigTest extends TestCase {
> +
> +
> +    public void testFixedWith() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals(false, config.isFixedWidth());
> +        config.setFixedWidth(true);
> +        assertEquals(true, config.isFixedWidth());
> +    }
> +
> +    public void testFields() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals(0, config.getFields().length);
> +        config.setFields((CSVField[])null);
> +        assertEquals(0, config.getFields().length);
> +        config.setFields((Collection)null);
> +        assertEquals(0, config.getFields().length);
> +        CSVField field = new CSVField();
> +        field.setName("field1");
> +        config.addField(field);
> +        assertEquals(field, config.getFields()[0]);
> +        assertEquals(null, config.getField(null));
> +        assertEquals(null, config.getField("field11"));
> +        assertEquals(field, config.getField("field1"));
> +    }
> +
> +    public void testFill() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals(CSVConfig.FILLNONE, config.getFill());
> +        config.setFill(CSVConfig.FILLLEFT);
> +        assertEquals(CSVConfig.FILLLEFT, config.getFill());
> +        config.setFill(CSVConfig.FILLRIGHT);
> +        assertEquals(CSVConfig.FILLRIGHT, config.getFill());
> +        assertEquals(' ', config.getFillChar());
> +        config.setFillChar('m');
> +        assertEquals('m', config.getFillChar());
> +    }
> +
> +    public void testDelimiter() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals(',', config.getDelimiter());
> +        config.setDelimiter(';');
> +        assertEquals(';', config.getDelimiter());
> +        assertEquals(false, config.isDelimiterIgnored());
> +        config.setIgnoreDelimiter(true);
> +        assertEquals(true, config.isDelimiterIgnored());
> +    }
> +
> +    public void testValueDelimiter() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals('"', config.getValueDelimiter());
> +        config.setValueDelimiter('m');
> +        assertEquals('m', config.getValueDelimiter());
> +        assertEquals(true, config.isValueDelimiterIgnored());
> +        config.setIgnoreValueDelimiter(false);
> +        assertEquals(false, config.isValueDelimiterIgnored());
> +    }
> +
> +    public void testFieldHeader() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals(false, config.isFieldHeader());
> +        config.setFieldHeader(true);
> +        assertEquals(true, config.isFieldHeader());
> +    }
> +
> +    public void testTrimEnd() {
> +        CSVConfig config = new CSVConfig();
> +        assertEquals(false, config.isEndTrimmed());
> +        config.setEndTrimmed(true);
> +        assertEquals(true, config.isEndTrimmed());
> +    }
> +
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,48 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.solr.internal.csv.writer;
> +
> +import junit.framework.TestCase;
> +
> +public class CSVFieldTest extends TestCase {
> +
> +    public void testCSVField() {
> +        CSVField field = new CSVField();
> +        assertEquals(null, field.getName());
> +        field.setName("id");
> +        assertEquals("id", field.getName());
> +        assertEquals(0, field.getSize());
> +        field.setSize(10);
> +        assertEquals(10, field.getSize());
> +        field = new CSVField("name");
> +        assertEquals("name", field.getName());
> +        field = new CSVField("name", 10);
> +        assertEquals("name", field.getName());
> +        assertEquals(10, field.getSize());
> +    }
> +
> +    public void testFill() {
> +        CSVField field = new CSVField();
> +        assertEquals(CSVConfig.FILLNONE, field.getFill());
> +        assertEquals(false, field.overrideFill());
> +        field.setFill(CSVConfig.FILLLEFT);
> +        assertEquals(true, field.overrideFill());
> +        assertEquals(CSVConfig.FILLLEFT, field.getFill());
> +    }
> +}
>
> Added: lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java?rev=1308225&view=auto
> ==============================================================================
> --- lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java (added)
> +++ lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java Mon Apr  2 01:29:20 2012
> @@ -0,0 +1,56 @@
> +/*
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements.  See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership.  The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License.  You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing,
> + * software distributed under the License is distributed on an
> + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> + * KIND, either express or implied.  See the License for the
> + * specific language governing permissions and limitations
> + * under the License.
> + */
> +package org.apache.solr.internal.csv.writer;
> +
> +import java.io.StringWriter;
> +import java.util.HashMap;
> +import java.util.Map;
> +
> +import junit.framework.TestCase;
> +
> +/**
> + * The testcase for the csv writer.
> + */
> +public class CSVWriterTest extends TestCase {
> +
> +    public void testCSVConfig() {
> +        CSVWriter writer = new CSVWriter();
> +        assertEquals(null, writer.getConfig());
> +        CSVConfig config = new CSVConfig();
> +        writer.setConfig(config);
> +        assertEquals(config, writer.getConfig());
> +        writer = new CSVWriter(config);
> +        assertEquals(config, writer.getConfig());
> +    }
> +
> +    public void testWriter() {
> +        CSVWriter writer = new CSVWriter();
> +        CSVConfig config = new CSVConfig();
> +        config.addField(new CSVField("field1", 5));
> +        config.addField(new CSVField("field2", 4));
> +        writer.setConfig(config);
> +        StringWriter sw = new StringWriter();
> +        writer.setWriter(sw);
> +        Map map = new HashMap();
> +        map.put("field1", "12345");
> +        map.put("field2", "1234");
> +        writer.writeRecord(map);
> +        assertEquals("12345,1234\n",sw.toString());
> +    }
> +}
>
>



-- 
lucidimagination.com

---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
For additional commands, e-mail: dev-help@lucene.apache.org

Re: svn commit: r1308225 - in /lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal: ./ csv/ csv/writer/

[Chris Male <ge...@gmail.com>]

Alrighty.

On Mon, Apr 2, 2012 at 11:29 PM, Robert Muir <rc...@gmail.com> wrote:

> Since i've already svn merge --reintegrated this branch, I don't think
> we can safely merge any further changes... I think any future
> development should be against trunk?
>
> "In Subversion 1.5, once a --reintegrate merge is done from branch to
> trunk, the branch is no longer usable for further work. It's not able
> to correctly absorb new trunk changes, nor can it be properly
> reintegrated to trunk again."
>
> http://svnbook.red-bean.com/en/1.5/svn.branchmerge.basicmerging.html
>
> On Sun, Apr 1, 2012 at 9:29 PM,  <ch...@apache.org> wrote:
> > Author: chrism
> > Date: Mon Apr  2 01:29:20 2012
> > New Revision: 1308225
> >
> > URL: http://svn.apache.org/viewvc?rev=1308225&view=rev
> > Log:
> > LUCENE-3930: Imported commons-csv tests
> >
> > Added:
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> >
>  lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVParserTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,581 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.IOException;
> > +import java.io.Reader;
> > +import java.io.StringReader;
> > +import java.util.Arrays;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * CSVParserTest
> > + *
> > + * The test are organized in three different sections:
> > + * The 'setter/getter' section, the lexer section and finally the parser
> > + * section. In case a test fails, you should follow a top-down approach
> for
> > + * fixing a potential bug (its likely that the parser itself fails if
> the lexer
> > + * has problems...).
> > + */
> > +public class CSVParserTest extends TestCase {
> > +
> > +  /**
> > +   * TestCSVParser.
> > +   */
> > +  class TestCSVParser extends CSVParser {
> > +    /**
> > +     * Test parser to investigate the type of the internal Token.
> > +     * @param in a Reader
> > +     */
> > +    TestCSVParser(Reader in) {
> > +      super(in);
> > +    }
> > +
> > +    TestCSVParser(Reader in, CSVStrategy strategy) {
> > +      super(in, strategy);
> > +    }
> > +    /**
> > +     * Calls super.nextToken() and prints out a String representation
> of token
> > +     * type and content.
> > +     * @return String representation of token type and content
> > +     * @throws IOException like {@link CSVParser#nextToken()}
> > +     */
> > +    public String testNextToken() throws IOException {
> > +      Token t = super.nextToken();
> > +      return Integer.toString(t.type) + ";" + t.content + ";";
> > +    }
> > +  }
> > +
> > +  // ======================================================
> > +  //   lexer tests
> > +  // ======================================================
> > +
> > +  // Single line (without comment)
> > +  public void testNextToken1() throws IOException {
> > +    String code = "abc,def, hijk,  lmnop,   qrst,uv ,wxy   ,z , ,";
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code));
> > +    assertEquals(CSVParser.TT_TOKEN + ";abc;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";def;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";hijk;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";lmnop;",
> parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";qrst;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";uv;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";wxy;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";z;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +  }
> > +
> > +  // multiline including comments (and empty lines)
> > +  public void testNextToken2() throws IOException {
> > +    /*   file:   1,2,3,
> > +     *           a,b x,c
> > +     *
> > +     *           # this is a comment
> > +     *           d,e,
> > +     *
> > +     */
> > +    String code = "1,2,3,\na,b x,c\n#foo\n\nd,e,\n\n";
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    // strategy.setIgnoreEmptyLines(false);
> > +    strategy.setCommentStart('#');
> > +
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code),
> strategy);
> > +
> > +
> > +    assertEquals(CSVParser.TT_TOKEN + ";1;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";2;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";3;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";b x;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";c;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";d;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";e;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +
> > +  }
> > +
> > +  // simple token with escaping
> > +  public void testNextToken3() throws IOException {
> > +    /* file: a,\,,b
> > +     *       \,,
> > +     */
> > +    String code = "a,\\,,b\n\\,,";
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setCommentStart('#');
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code),
> strategy);
> > +
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    // an unquoted single backslash is not an escape char
> > +    assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> > +    // an unquoted single backslash is not an escape char
> > +    assertEquals(CSVParser.TT_TOKEN + ";\\;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";;", parser.testNextToken());
> > +  }
> > +
> > +  // encapsulator tokenizer (sinle line)
> > +  public void testNextToken4() throws IOException {
> > +    /* file:  a,"foo",b
> > +     *        a,   " foo",b
> > +     *        a,"foo "   ,b     // whitespace after closing encapsulator
> > +     *        a,  " foo " ,b
> > +     */
> > +     String code =
> > +      "a,\"foo\",b\na,   \" foo\",b\na,\"foo \"  ,b\na,  \" foo \"  ,b";
> > +     TestCSVParser parser = new TestCSVParser(new StringReader(code));
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";foo;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + "; foo;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";foo ;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +     assertEquals(CSVParser.TT_TOKEN + "; foo ;",
> parser.testNextToken());
> > +//     assertEquals(CSVParser.TT_EORECORD + ";b;",
> parser.testNextToken());
> > +     assertEquals(CSVParser.TT_EOF + ";b;", parser.testNextToken());
> > +  }
> > +
> > +  // encapsulator tokenizer (multi line, delimiter in string)
> > +  public void testNextToken5() throws IOException {
> > +    String code =
> > +      "a,\"foo\n\",b\n\"foo\n  baar ,,,\"\n\"\n\t \n\"";
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code));
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_TOKEN + ";foo\n;",
> parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";b;", parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EORECORD + ";foo\n  baar ,,,;",
> > +        parser.testNextToken());
> > +    assertEquals(CSVParser.TT_EOF + ";\n\t \n;",
> parser.testNextToken());
> > +
> > +  }
> > +
> > +  // change delimiters, comment, encapsulater
> > +  public void testNextToken6() throws IOException {
> > +    /* file: a;'b and \' more
> > +     *       '
> > +     *       !comment;;;;
> > +     *       ;;
> > +     */
> > +    String code = "a;'b and '' more\n'\n!comment;;;;\n;;";
> > +    TestCSVParser parser = new TestCSVParser(new StringReader(code),
> new CSVStrategy(';', '\'', '!'));
> > +    assertEquals(CSVParser.TT_TOKEN + ";a;", parser.testNextToken());
> > +    assertEquals(
> > +      CSVParser.TT_EORECORD + ";b and ' more\n;",
> > +      parser.testNextToken());
> > +  }
> > +
> > +
> > +  // ======================================================
> > +  //   parser tests
> > +  // ======================================================
> > +
> > +  String code =
> > +    "a,b,c,d\n"
> > +    + " a , b , 1 2 \n"
> > +    + "\"foo baar\", b,\n"
> > +   // + "   \"foo\n,,\n\"\",,\n\\\"\",d,e\n";
> > +      + "   \"foo\n,,\n\"\",,\n\"\"\",d,e\n";   // changed to use
> standard CSV escaping
> > +  String[][] res = {
> > +    {"a", "b", "c", "d"},
> > +    {"a", "b", "1 2"},
> > +    {"foo baar", "b", ""},
> > +    {"foo\n,,\n\",,\n\"", "d", "e"}
> > +  };
> > +  public void testGetLine() throws IOException {
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String[] tmp = null;
> > +    for (int i = 0; i < res.length; i++) {
> > +      tmp = parser.getLine();
> > +      assertTrue(Arrays.equals(res[i], tmp));
> > +    }
> > +    tmp = parser.getLine();
> > +    assertTrue(tmp == null);
> > +  }
> > +
> > +  public void testNextValue() throws IOException {
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String tmp = null;
> > +    for (int i = 0; i < res.length; i++) {
> > +      for (int j = 0; j < res[i].length; j++) {
> > +        tmp = parser.nextValue();
> > +        assertEquals(res[i][j], tmp);
> > +      }
> > +    }
> > +    tmp = parser.nextValue();
> > +    assertTrue(tmp == null);
> > +  }
> > +
> > +  public void testGetAllValues() throws IOException {
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testExcelStrategy1() throws IOException {
> > +    String code =
> > +      "value1,value2,value3,value4\r\na,b,c,d\r\n  x,,,"
> > +      + "\r\n\r\n\"\"\"hello\"\"\",\"
>  \"\"world\"\"\",\"abc\ndef\",\r\n";
> > +    String[][] res = {
> > +      {"value1", "value2", "value3", "value4"},
> > +      {"a", "b", "c", "d"},
> > +      {"  x", "", "", ""},
> > +      {""},
> > +      {"\"hello\"", "  \"world\"", "abc\ndef", ""}
> > +    };
> > +    CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testExcelStrategy2() throws Exception {
> > +    String code = "foo,baar\r\n\r\nhello,\r\n\r\nworld,\r\n";
> > +    String[][] res = {
> > +      {"foo", "baar"},
> > +      {""},
> > +      {"hello", ""},
> > +      {""},
> > +      {"world", ""}
> > +    };
> > +    CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testEndOfFileBehaviourExcel() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\nworld,\r\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\r\n",
> > +        "hello,\r\n\r\nworld,\"\"",
> > +        "hello,\r\n\r\nworld,\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\n",
> > +        "hello,\r\n\r\nworld,\"\""
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""},
> > +      {""},  // ExcelStrategy does not ignore empty lines
> > +      {"world", ""}
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void testEndOfFileBehaviorCSV() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\nworld,\r\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\r\n",
> > +        "hello,\r\n\r\nworld,\"\"",
> > +        "hello,\r\n\r\nworld,\n",
> > +        "hello,\r\n\r\nworld,",
> > +        "hello,\r\n\r\nworld,\"\"\n",
> > +        "hello,\r\n\r\nworld,\"\""
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""},  // CSV Strategy ignores empty lines
> > +      {"world", ""}
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void testEmptyLineBehaviourExcel() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\n\r\n",
> > +        "hello,\n\n\n",
> > +        "hello,\"\"\r\n\r\n\r\n",
> > +        "hello,\"\"\n\n\n"
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""},
> > +      {""},  // ExcelStrategy does not ignore empty lines
> > +      {""}
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code),
> CSVStrategy.EXCEL_STRATEGY);
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void testEmptyLineBehaviourCSV() throws Exception {
> > +    String[] codes = {
> > +        "hello,\r\n\r\n\r\n",
> > +        "hello,\n\n\n",
> > +        "hello,\"\"\r\n\r\n\r\n",
> > +        "hello,\"\"\n\n\n"
> > +        };
> > +    String[][] res = {
> > +      {"hello", ""}  // CSV Strategy ignores empty lines
> > +    };
> > +    String code;
> > +    for (int codeIndex = 0; codeIndex < codes.length; codeIndex++) {
> > +      code = codes[codeIndex];
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] tmp = parser.getAllValues();
> > +      assertEquals(res.length, tmp.length);
> > +      assertTrue(tmp.length > 0);
> > +      for (int i = 0; i < res.length; i++) {
> > +        assertTrue(Arrays.equals(res[i], tmp[i]));
> > +      }
> > +    }
> > +  }
> > +
> > +  public void OLDtestBackslashEscaping() throws IOException {
> > +    String code =
> > +      "one,two,three\n"
> > +      + "on\\\"e,two\n"
> > +      + "on\"e,two\n"
> > +      + "one,\"tw\\\"o\"\n"
> > +      + "one,\"t\\,wo\"\n"
> > +      + "one,two,\"th,ree\"\n"
> > +      + "\"a\\\\\"\n"
> > +      + "a\\,b\n"
> > +      + "\"a\\\\,b\"";
> > +    String[][] res = {
> > +        { "one", "two", "three" },
> > +        { "on\\\"e", "two" },
> > +        { "on\"e", "two" },
> > +        { "one", "tw\"o" },
> > +        { "one", "t\\,wo" },  // backslash in quotes only escapes a
> delimiter (",")
> > +        { "one", "two", "th,ree" },
> > +        { "a\\\\" },     // backslash in quotes only escapes a
> delimiter (",")
> > +        { "a\\", "b" },  // a backslash must be returnd
> > +        { "a\\\\,b" }    // backslash in quotes only escapes a
> delimiter (",")
> > +      };
> > +    CSVParser parser = new CSVParser(new StringReader(code));
> > +    String[][] tmp = parser.getAllValues();
> > +    assertEquals(res.length, tmp.length);
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testBackslashEscaping() throws IOException {
> > +
> > +    // To avoid confusion over the need for escaping chars in java code,
> > +    // We will test with a forward slash as the escape char, and a
> single
> > +    // quote as the encapsulator.
> > +
> > +    String code =
> > +      "one,two,three\n" // 0
> > +      + "'',''\n"       // 1) empty encapsulators
> > +      + "/',/'\n"       // 2) single encapsulators
> > +      + "'/'','/''\n"   // 3) single encapsulators encapsulated via
> escape
> > +      + "'''',''''\n"   // 4) single encapsulators encapsulated via
> doubling
> > +      + "/,,/,\n"       // 5) separator escaped
> > +      + "//,//\n"       // 6) escape escaped
> > +      + "'//','//'\n"   // 7) escape escaped in encapsulation
> > +      + "   8   ,   \"quoted \"\" /\" // string\"   \n"     // don't
> eat spaces
> > +      + "9,   /\n   \n"  // escaped newline
> > +      + "";
> > +    String[][] res = {
> > +        { "one", "two", "three" }, // 0
> > +        { "", "" },                // 1
> > +        { "'", "'" },              // 2
> > +        { "'", "'" },              // 3
> > +        { "'", "'" },              // 4
> > +        { ",", "," },              // 5
> > +        { "/", "/" },              // 6
> > +        { "/", "/" },              // 7
> > +        { "   8   ", "   \"quoted \"\" \" / string\"   " },
> > +        { "9", "   \n   " },
> > +      };
> > +
> > +
> > +    CSVStrategy strategy = new
> CSVStrategy(',','\'',CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true);
> > +
> > +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertTrue(tmp.length > 0);
> > +    for (int i = 0; i < res.length; i++) {
> > +      assertTrue(Arrays.equals(res[i], tmp[i]));
> > +    }
> > +  }
> > +
> > +  public void testBackslashEscaping2() throws IOException {
> > +
> > +    // To avoid confusion over the need for escaping chars in java code,
> > +    // We will test with a forward slash as the escape char, and a
> single
> > +    // quote as the encapsulator.
> > +
> > +    String code = ""
> > +      + " , , \n"           // 1)
> > +      + " \t ,  , \n"       // 2)
> > +      + " // , /, , /,\n"   // 3)
> > +      + "";
> > +    String[][] res = {
> > +        { " ", " ", " " },         // 1
> > +        { " \t ", "  ", " " },         // 2
> > +        { " / ", " , ", " ," },         //3
> > +      };
> > +
> > +
> > +    CSVStrategy strategy = new
> CSVStrategy(',',CSVStrategy.ENCAPSULATOR_DISABLED,CSVStrategy.COMMENTS_DISABLED,'/',false,false,true,true);
> > +
> > +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertTrue(tmp.length > 0);
> > +
> > +    if (!CSVPrinterTest.equals(res, tmp)) {
> > +      assertTrue(false);
> > +    }
> > +
> > +  }
> > +
> > +
> > +  public void testDefaultStrategy() throws IOException {
> > +
> > +    String code = ""
> > +        + "a,b\n"            // 1)
> > +        + "\"\n\",\" \"\n"   // 2)
> > +        + "\"\",#\n"   // 2)
> > +        ;
> > +    String[][] res = {
> > +        { "a", "b" },
> > +        { "\n", " " },
> > +        { "", "#" },
> > +    };
> > +
> > +    CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
> > +    assertEquals(CSVStrategy.COMMENTS_DISABLED,
> strategy.getCommentStart());
> > +
> > +    CSVParser parser = new CSVParser(new StringReader(code), strategy);
> > +    String[][] tmp = parser.getAllValues();
> > +    assertTrue(tmp.length > 0);
> > +
> > +    if (!CSVPrinterTest.equals(res, tmp)) {
> > +      assertTrue(false);
> > +    }
> > +
> > +    String[][] res_comments = {
> > +        { "a", "b" },
> > +        { "\n", " " },
> > +        { ""},
> > +    };
> > +
> > +    strategy = new CSVStrategy(',','"','#');
> > +    parser = new CSVParser(new StringReader(code), strategy);
> > +    tmp = parser.getAllValues();
> > +
> > +    if (!CSVPrinterTest.equals(res_comments, tmp)) {
> > +      assertTrue(false);
> > +    }
> > +  }
> > +
> > +
> > +    public void testUnicodeEscape() throws IOException {
> > +      String code = "abc,\\u0070\\u0075\\u0062\\u006C\\u0069\\u0063";
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      parser.getStrategy().setUnicodeEscapeInterpretation(true);
> > +      String[] data = parser.getLine();
> > +      assertEquals(2, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("public", data[1]);
> > +    }
> > +
> > +    public void testCarriageReturnLineFeedEndings() throws IOException {
> > +     String code = "foo\r\nbaar,\r\nhello,world\r\n,kanu";
> > +     CSVParser parser = new CSVParser(new StringReader(code));
> > +     String[][] data = parser.getAllValues();
> > +     assertEquals(4, data.length);
> > +    }
> > +
> > +    public void testIgnoreEmptyLines() throws IOException {
> > +      String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
> > +      //String code = "world\r\n\n";
> > +      //String code = "foo;baar\r\n\r\nhello;\r\n\r\nworld;\r\n";
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] data = parser.getAllValues();
> > +      assertEquals(3, data.length);
> > +    }
> > +
> > +    public void testLineTokenConsistency() throws IOException {
> > +      String code = "\nfoo,baar\n\r\n,\n\n,world\r\n\n";
> > +      CSVParser parser = new CSVParser(new StringReader(code));
> > +      String[][] data = parser.getAllValues();
> > +      parser = new CSVParser(new StringReader(code));
> > +      CSVParser parser1 = new CSVParser(new StringReader(code));
> > +      for (int i = 0; i < data.length; i++) {
> > +        assertTrue(Arrays.equals(parser1.getLine(), data[i]));
> > +        for (int j = 0; j < data[i].length; j++) {
> > +          assertEquals(parser.nextValue(), data[i][j]);
> > +        }
> > +      }
> > +    }
> > +
> > +    // From SANDBOX-153
> > +     public void testDelimiterIsWhitespace() throws IOException {
> > +         String code = "one\ttwo\t\tfour \t five\t six";
> > +         TestCSVParser parser = new TestCSVParser(new
> StringReader(code), CSVStrategy.TDF_STRATEGY);
> > +         assertEquals(CSVParser.TT_TOKEN + ";one;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";two;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";four;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_TOKEN + ";five;",
> parser.testNextToken());
> > +         assertEquals(CSVParser.TT_EOF + ";six;",
> parser.testNextToken());
> > +     }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVPrinterTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,197 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.IOException;
> > +import java.io.StringReader;
> > +import java.io.StringWriter;
> > +import java.util.Arrays;
> > +import java.util.Random;
> > +
> > +import junit.framework.Test;
> > +import junit.framework.TestCase;
> > +import junit.framework.TestSuite;
> > +
> > +/**
> > + * CSVPrinterTest
> > + */
> > +public class CSVPrinterTest extends TestCase {
> > +
> > +  String lineSeparator = "\n";
> > +
> > +  public void testPrinter1() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw,
> CSVStrategy.DEFAULT_STRATEGY);
> > +    String[] line1 = {"a", "b"};
> > +    printer.println(line1);
> > +    assertEquals("a,b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testPrinter2() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw,
> CSVStrategy.DEFAULT_STRATEGY);
> > +    String[] line1 = {"a,b", "b"};
> > +    printer.println(line1);
> > +    assertEquals("\"a,b\",b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testPrinter3() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw,
> CSVStrategy.DEFAULT_STRATEGY);
> > +    String[] line1 = {"a, b", "b "};
> > +    printer.println(line1);
> > +    assertEquals("\"a, b\",\"b \"" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testExcelPrinter1() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.EXCEL_STRATEGY);
> > +    String[] line1 = {"a", "b"};
> > +    printer.println(line1);
> > +    assertEquals("a,b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +  public void testExcelPrinter2() throws IOException {
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw, CSVStrategy.EXCEL_STRATEGY);
> > +    String[] line1 = {"a,b", "b"};
> > +    printer.println(line1);
> > +    assertEquals("\"a,b\",b" + lineSeparator, sw.toString());
> > +  }
> > +
> > +
> > +
> > +  public void testRandom() throws Exception {
> > +    int iter=10000;
> > +    strategy = CSVStrategy.DEFAULT_STRATEGY;
> > +    doRandom(iter);
> > +    strategy = CSVStrategy.EXCEL_STRATEGY;
> > +    doRandom(iter);
> > +
> > +    // Strategy for MySQL
> > +    strategy = new CSVStrategy('\t', CSVStrategy.ENCAPSULATOR_DISABLED,
> CSVStrategy.COMMENTS_DISABLED,'\\',false, false, false, false);
> > +    doRandom(iter);
> > +  }
> > +
> > +  Random r = new Random();
> > +  CSVStrategy strategy;
> > +
> > +  public void doRandom(int iter) throws Exception {
> > +    for (int i=0; i<iter; i++) {
> > +      doOneRandom();
> > +    }
> > +  }
> > +
> > +  public void doOneRandom() throws Exception {
> > +    int nLines = r.nextInt(4)+1;
> > +    int nCol = r.nextInt(3)+1;
> > +    // nLines=1;nCol=2;
> > +    String[][] lines = new String[nLines][];
> > +    for (int i=0; i<nLines; i++) {
> > +      String[] line = new String[nCol];
> > +      lines[i] = line;
> > +      for (int j=0; j<nCol; j++) {
> > +        line[j] = randStr();
> > +      }
> > +    }
> > +
> > +    StringWriter sw = new StringWriter();
> > +    CSVPrinter printer = new CSVPrinter(sw, strategy);
> > +
> > +    for (int i=0; i<nLines; i++) {
> > +      // for (int j=0; j<lines[i].length; j++) System.out.println("###
> VALUE=:" + printable(lines[i][j]));
> > +      printer.println(lines[i]);
> > +    }
> > +
> > +    printer.flush();
> > +    String result = sw.toString();
> > +    // System.out.println("### :" + printable(result));
> > +
> > +    StringReader reader = new StringReader(result);
> > +
> > +    CSVParser parser = new CSVParser(reader, strategy);
> > +    String[][] parseResult = parser.getAllValues();
> > +
> > +    if (!equals(lines, parseResult)) {
> > +      System.out.println("Printer output :" + printable(result));
> > +      assertTrue(false);
> > +    }
> > +  }
> > +
> > +  public static boolean equals(String[][] a, String[][] b) {
> > +    if (a.length != b.length) {
> > +      return false;
> > +    }
> > +    for (int i=0; i<a.length; i++) {
> > +      String[] linea = a[i];
> > +      String[] lineb = b[i];
> > +      if (linea.length != lineb.length) {
> > +        return false;
> > +      }
> > +      for (int j=0; j<linea.length; j++) {
> > +        String aval = linea[j];
> > +        String bval = lineb[j];
> > +        if (!aval.equals(bval)) {
> > +          System.out.println("expected  :" + printable(aval));
> > +          System.out.println("got       :" + printable(bval));
> > +          return false;
> > +        }
> > +      }
> > +    }
> > +    return true;
> > +  }
> > +
> > +  public static String printable(String s) {
> > +    StringBuffer sb = new StringBuffer();
> > +    for (int i=0; i<s.length(); i++) {
> > +      char ch = s.charAt(i);
> > +      if (ch<=' ' || ch>=128) {
> > +        sb.append("(" + (int)ch + ")");
> > +      } else {
> > +        sb.append(ch);
> > +      }
> > +    }
> > +    return sb.toString();
> > +  }
> > +
> > +  public String randStr() {
> > +    int sz = r.nextInt(20);
> > +    // sz = r.nextInt(3);
> > +    char[] buf = new char[sz];
> > +    for (int i=0; i<sz; i++) {
> > +      // stick in special chars with greater frequency
> > +      char ch;
> > +      int what = r.nextInt(20);
> > +      switch (what) {
> > +        case 0: ch = '\r'; break;
> > +        case 1: ch = '\n'; break;
> > +        case 2: ch = '\t'; break;
> > +        case 3: ch = '\f'; break;
> > +        case 4: ch = ' ';  break;
> > +        case 5: ch = ',';  break;
> > +        case 6: ch = '"';  break;
> > +        case 7: ch = '\''; break;
> > +        case 8: ch = '\\'; break;
> > +        default: ch = (char)r.nextInt(300); break;
> > +        // default: ch = 'a'; break;
> > +      }
> > +      buf[i] = ch;
> > +    }
> > +    return new String(buf);
> > +  }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVStrategyTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,91 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.StringReader;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * CSVStrategyTest
> > + *
> > + * The test are organized in three different sections:
> > + * The 'setter/getter' section, the lexer section and finally the
> strategy
> > + * section. In case a test fails, you should follow a top-down approach
> for
> > + * fixing a potential bug (its likely that the strategy itself fails if
> the lexer
> > + * has problems...).
> > + */
> > +public class CSVStrategyTest extends TestCase {
> > +
> > +  // ======================================================
> > +  //   getters / setters
> > +  // ======================================================
> > +  public void testGetSetCommentStart() {
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setCommentStart('#');
> > +    assertEquals(strategy.getCommentStart(), '#');
> > +    strategy.setCommentStart('!');
> > +    assertEquals(strategy.getCommentStart(), '!');
> > +  }
> > +
> > +  public void testGetSetEncapsulator() {
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setEncapsulator('"');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    strategy.setEncapsulator('\'');
> > +    assertEquals(strategy.getEncapsulator(), '\'');
> > +  }
> > +
> > +  public void testGetSetDelimiter() {
> > +    CSVStrategy strategy =
> (CSVStrategy)CSVStrategy.DEFAULT_STRATEGY.clone();
> > +    strategy.setDelimiter(';');
> > +    assertEquals(strategy.getDelimiter(), ';');
> > +    strategy.setDelimiter(',');
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    strategy.setDelimiter('\t');
> > +    assertEquals(strategy.getDelimiter(), '\t');
> > +  }
> > +
> > +  public void testSetCSVStrategy() {
> > +    CSVStrategy strategy = CSVStrategy.DEFAULT_STRATEGY;
> > +    // default settings
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    assertEquals(strategy.getCommentStart(),
> CSVStrategy.COMMENTS_DISABLED);
> > +    assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
> > +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> > +    assertEquals(true,  strategy.getIgnoreEmptyLines());
> > +    // explicit csv settings
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    assertEquals(strategy.getCommentStart(),
> CSVStrategy.COMMENTS_DISABLED);
> > +    assertEquals(true,  strategy.getIgnoreLeadingWhitespaces());
> > +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> > +    assertEquals(true,  strategy.getIgnoreEmptyLines());
> > +  }
> > +
> > +  public void testSetExcelStrategy() {
> > +    CSVStrategy strategy = CSVStrategy.EXCEL_STRATEGY;
> > +    assertEquals(strategy.getDelimiter(), ',');
> > +    assertEquals(strategy.getEncapsulator(), '"');
> > +    assertEquals(strategy.getCommentStart(),
> CSVStrategy.COMMENTS_DISABLED);
> > +    assertEquals(false,  strategy.getIgnoreLeadingWhitespaces());
> > +    assertEquals(false, strategy.getUnicodeEscapeInterpretation());
> > +    assertEquals(false, strategy.getIgnoreEmptyLines());
> > +  }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CSVUtilsTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,150 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.IOException;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * CSVUtilsTest
> > + */
> > +public class CSVUtilsTest extends TestCase {
> > +
> > +  // ======================================================
> > +  //   static parser tests
> > +  // ======================================================
> > +  public void testParse1() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc\ndef");
> > +      assertEquals(2, data.length);
> > +      assertEquals(1, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def", data[1][0]);
> > +    }
> > +
> > +    public void testParse2() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,def,\"ghi,jkl\"\ndef");
> > +      assertEquals(2, data.length);
> > +      assertEquals(3, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def", data[0][1]);
> > +      assertEquals("ghi,jkl", data[0][2]);
> > +      assertEquals("def", data[1][0]);
> > +    }
> > +
> > +    public void testParse3() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,\"def\nghi\"\njkl");
> > +      assertEquals(2, data.length);
> > +      assertEquals(2, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def\nghi", data[0][1]);
> > +      assertEquals("jkl", data[1][0]);
> > +    }
> > +
> > +    public void testParse4() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,\"def\\\\nghi\"\njkl");
> > +      assertEquals(2, data.length);
> > +      assertEquals(2, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      // an escape char in quotes only escapes a delimiter, not itself
> > +      assertEquals("def\\\\nghi", data[0][1]);
> > +      assertEquals("jkl", data[1][0]);
> > +    }
> > +
> > +    public void testParse5() throws IOException {
> > +      String[][] data = CSVUtils.parse("abc,def\\nghi\njkl");
> > +      assertEquals(2, data.length);
> > +      assertEquals(2, data[0].length);
> > +      assertEquals(1, data[1].length);
> > +      assertEquals("abc", data[0][0]);
> > +      assertEquals("def\\nghi", data[0][1]);
> > +      assertEquals("jkl", data[1][0]);
> > +    }
> > +
> > +    public void testParse6() throws IOException {
> > +      String[][] data = CSVUtils.parse("");
> > +      // default strategy is CSV, which ignores empty lines
> > +      assertEquals(0, data.length);
> > +    }
> > +
> > +    public void testParse7() throws IOException {
> > +      boolean io = false;
> > +      try {
> > +        CSVUtils.parse(null);
> > +      } catch (IllegalArgumentException e) {
> > +        io = true;
> > +      }
> > +      assertTrue(io);
> > +    }
> > +
> > +    public void testParseLine1() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,def,ghi");
> > +      assertEquals(3, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def", data[1]);
> > +      assertEquals("ghi", data[2]);
> > +    }
> > +
> > +    public void testParseLine2() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,def,ghi\n");
> > +      assertEquals(3, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def", data[1]);
> > +      assertEquals("ghi", data[2]);
> > +    }
> > +
> > +    public void testParseLine3() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,\"def,ghi\"");
> > +      assertEquals(2, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def,ghi", data[1]);
> > +    }
> > +
> > +    public void testParseLine4() throws IOException {
> > +      String[] data = CSVUtils.parseLine("abc,\"def\nghi\"");
> > +      assertEquals(2, data.length);
> > +      assertEquals("abc", data[0]);
> > +      assertEquals("def\nghi", data[1]);
> > +    }
> > +
> > +    public void testParseLine5() throws IOException {
> > +      String[] data = CSVUtils.parseLine("");
> > +      assertEquals(0, data.length);
> > +      // assertEquals("", data[0]);
> > +    }
> > +
> > +    public void testParseLine6() throws IOException {
> > +      boolean io = false;
> > +      try {
> > +        CSVUtils.parseLine(null);
> > +      } catch (IllegalArgumentException e) {
> > +        io = true;
> > +      }
> > +      assertTrue(io);
> > +    }
> > +
> > +    public void testParseLine7() throws IOException {
> > +      String[] res = CSVUtils.parseLine("");
> > +      assertNotNull(res);
> > +      assertEquals(0, res.length);
> > +    }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/CharBufferTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,192 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import junit.framework.TestCase;
> > +
> > +public class CharBufferTest extends TestCase {
> > +    public void testCreate() {
> > +        CharBuffer cb = new CharBuffer();
> > +        assertEquals(0, cb.length());
> > +        try {
> > +            cb = new CharBuffer(0);
> > +            fail("Should not be possible");
> > +        } catch(IllegalArgumentException e) {
> > +            // expected
> > +        }
> > +
> > +        cb = new CharBuffer(128);
> > +        assertEquals(0, cb.length());
> > +    }
> > +
> > +    public void testAppendChar() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        String expected = "";
> > +        for (char c = 'a'; c < 'z'; c++) {
> > +            cb.append(c);
> > +            expected += c;
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(expected.length(), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendCharArray() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        char[] abcd = "abcd".toCharArray();
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += "abcd";
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendString() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        String abcd = "abcd";
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += abcd;
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendStringBuffer() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        StringBuffer abcd = new StringBuffer("abcd");
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += "abcd";
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testAppendCharBuffer() {
> > +        CharBuffer cb = new CharBuffer(1);
> > +        CharBuffer abcd = new CharBuffer(17);
> > +        abcd.append("abcd");
> > +        String expected = "";
> > +        for (int i=0; i<10; i++) {
> > +            cb.append(abcd);
> > +            expected += "abcd";
> > +            assertEquals(expected, cb.toString());
> > +            assertEquals(4*(i+1), cb.length());
> > +        }
> > +    }
> > +
> > +    public void testShrink() {
> > +        String data = "123456789012345678901234567890";
> > +
> > +        CharBuffer cb = new CharBuffer(data.length() + 100);
> > +        assertEquals(data.length() + 100, cb.capacity());
> > +        cb.append(data);
> > +        assertEquals(data.length() + 100, cb.capacity());
> > +        assertEquals(data.length(), cb.length());
> > +        cb.shrink();
> > +        assertEquals(data.length(), cb.capacity());
> > +        assertEquals(data.length(), cb.length());
> > +        assertEquals(data, cb.toString());
> > +    }
> > +
> > +    //-- the following test cases have been adapted from the
> HttpComponents project
> > +    //-- written by Oleg Kalnichevski
> > +
> > +    public void testSimpleAppend() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(16);
> > +        assertEquals(16, buffer.capacity());
> > +        assertEquals(0, buffer.length());
> > +        char[] b1 = buffer.getCharacters();
> > +        assertNotNull(b1);
> > +        assertEquals(0, b1.length);
> > +        assertEquals(0, buffer.length());
> > +
> > +        char[] tmp = new char[] { '1', '2', '3', '4'};
> > +        buffer.append(tmp);
> > +        assertEquals(16, buffer.capacity());
> > +        assertEquals(4, buffer.length());
> > +
> > +        char[] b2 = buffer.getCharacters();
> > +        assertNotNull(b2);
> > +        assertEquals(4, b2.length);
> > +        for (int i = 0; i < tmp.length; i++) {
> > +            assertEquals(tmp[i], b2[i]);
> > +        }
> > +        assertEquals("1234", buffer.toString());
> > +
> > +        buffer.clear();
> > +        assertEquals(16, buffer.capacity());
> > +        assertEquals(0, buffer.length());
> > +    }
> > +
> > +    public void testAppendString2() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(8);
> > +        buffer.append("stuff");
> > +        buffer.append(" and more stuff");
> > +        assertEquals("stuff and more stuff", buffer.toString());
> > +    }
> > +
> > +    public void testAppendNull() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(8);
> > +
> > +        buffer.append((StringBuffer)null);
> > +        assertEquals("", buffer.toString());
> > +
> > +        buffer.append((String)null);
> > +        assertEquals("", buffer.toString());
> > +
> > +        buffer.append((CharBuffer)null);
> > +        assertEquals("", buffer.toString());
> > +
> > +        buffer.append((char[])null);
> > +        assertEquals("", buffer.toString());
> > +    }
> > +
> > +    public void testAppendCharArrayBuffer() throws Exception {
> > +        CharBuffer buffer1 = new CharBuffer(8);
> > +        buffer1.append(" and more stuff");
> > +        CharBuffer buffer2 = new CharBuffer(8);
> > +        buffer2.append("stuff");
> > +        buffer2.append(buffer1);
> > +        assertEquals("stuff and more stuff", buffer2.toString());
> > +    }
> > +
> > +    public void testAppendSingleChar() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(4);
> > +        buffer.append('1');
> > +        buffer.append('2');
> > +        buffer.append('3');
> > +        buffer.append('4');
> > +        buffer.append('5');
> > +        buffer.append('6');
> > +        assertEquals("123456", buffer.toString());
> > +    }
> > +
> > +    public void testProvideCapacity() throws Exception {
> > +        CharBuffer buffer = new CharBuffer(4);
> > +        buffer.provideCapacity(2);
> > +        assertEquals(4, buffer.capacity());
> > +        buffer.provideCapacity(8);
> > +        assertTrue(buffer.capacity() >= 8);
> > +    }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/ExtendedBufferedReaderTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,222 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one or more
> > + * contributor license agreements.  See the NOTICE file distributed with
> > + * this work for additional information regarding copyright ownership.
> > + * The ASF licenses this file to You under the Apache License, Version
> 2.0
> > + * (the "License"); you may not use this file except in compliance with
> > + * the License.  You may obtain a copy of the License at
> > + *
> > + *      http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing, software
> > + * distributed under the License is distributed on an "AS IS" BASIS,
> > + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
> implied.
> > + * See the License for the specific language governing permissions and
> > + * limitations under the License.
> > + */
> > +package org.apache.solr.internal.csv;
> > +
> > +import java.io.StringReader;
> > +import java.util.Arrays;
> > +
> > +import junit.framework.Test;
> > +import junit.framework.TestCase;
> > +import junit.framework.TestSuite;
> > +
> > +/**
> > + * ExtendedBufferedReaderTest
> > + *
> > + */
> > +public class ExtendedBufferedReaderTest extends TestCase {
> > +
> > +  // ======================================================
> > +  //   the test cases
> > +  // ======================================================
> > +
> > +  public void testConstructors() {
> > +    ExtendedBufferedReader br = new ExtendedBufferedReader(new
> StringReader(""));
> > +    br = new ExtendedBufferedReader(new StringReader(""), 10);
> > +  }
> > +
> > +  public void testReadLookahead1() throws Exception {
> > +
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM,
> getEBR("").read());
> > +    ExtendedBufferedReader br = getEBR("1\n2\r3\n");
> > +    assertEquals('1', br.lookAhead());
> > +    assertEquals(ExtendedBufferedReader.UNDEFINED, br.readAgain());
> > +    assertEquals('1', br.read());
> > +    assertEquals('1', br.readAgain());
> > +
> > +    assertEquals(0, br.getLineNumber());
> > +    assertEquals('\n', br.lookAhead());
> > +    assertEquals(0, br.getLineNumber());
> > +    assertEquals('1', br.readAgain());
> > +    assertEquals('\n', br.read());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(1, br.getLineNumber());
> > +
> > +    assertEquals('2', br.lookAhead());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('2', br.read());
> > +    assertEquals('2', br.readAgain());
> > +
> > +    assertEquals('\r', br.lookAhead());
> > +    assertEquals('2', br.readAgain());
> > +    assertEquals('\r', br.read());
> > +    assertEquals('\r', br.readAgain());
> > +
> > +    assertEquals('3', br.lookAhead());
> > +    assertEquals('\r', br.readAgain());
> > +    assertEquals('3', br.read());
> > +    assertEquals('3', br.readAgain());
> > +
> > +    assertEquals('\n', br.lookAhead());
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('3', br.readAgain());
> > +    assertEquals('\n', br.read());
> > +    assertEquals(2, br.getLineNumber());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(2, br.getLineNumber());
> > +
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.lookAhead());
> > +    assertEquals('\n', br.readAgain());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.read());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.readAgain());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.read());
> > +    assertEquals(ExtendedBufferedReader.END_OF_STREAM, br.lookAhead());
> > +
> > +  }
> > +
> > +
> > +  public void testReadLookahead2() throws Exception {
> > +    char[] ref = new char[5];
> > +    char[] res = new char[5];
> > +
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertEquals(0, br.read(res, 0, 0));
> > +    assertTrue(Arrays.equals(res, ref));
> > +
> > +    br = getEBR("abcdefg");
> > +    ref[0] = 'a';
> > +    ref[1] = 'b';
> > +    ref[2] = 'c';
> > +    assertEquals(3, br.read(res, 0, 3));
> > +    assertTrue(Arrays.equals(res, ref));
> > +    assertEquals('c', br.readAgain());
> > +
> > +    assertEquals('d', br.lookAhead());
> > +    ref[4] = 'd';
> > +    assertEquals(1, br.read(res, 4, 1));
> > +    assertTrue(Arrays.equals(res, ref));
> > +    assertEquals('d', br.readAgain());
> > +
> > +  }
> > +
> > +  public void testMarkSupported() {
> > +    assertFalse(getEBR("foo").markSupported());
> > +  }
> > +
> > +  public void testReadLine() throws Exception {
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertTrue(br.readLine() == null);
> > +
> > +    br = getEBR("\n");
> > +    assertTrue(br.readLine().equals(""));
> > +    assertTrue(br.readLine() == null);
> > +
> > +    br = getEBR("foo\n\nhello");
> > +    assertEquals(0, br.getLineNumber());
> > +    assertTrue(br.readLine().equals("foo"));
> > +    assertEquals(1, br.getLineNumber());
> > +    assertTrue(br.readLine().equals(""));
> > +    assertEquals(2, br.getLineNumber());
> > +    assertTrue(br.readLine().equals("hello"));
> > +    assertEquals(3, br.getLineNumber());
> > +    assertTrue(br.readLine() == null);
> > +    assertEquals(3, br.getLineNumber());
> > +
> > +    br = getEBR("foo\n\nhello");
> > +    assertEquals('f', br.read());
> > +    assertEquals('o', br.lookAhead());
> > +    assertTrue(br.readLine().equals("oo"));
> > +    assertEquals(1, br.getLineNumber());
> > +    assertEquals('\n', br.lookAhead());
> > +    assertTrue(br.readLine().equals(""));
> > +    assertEquals(2, br.getLineNumber());
> > +    assertEquals('h', br.lookAhead());
> > +    assertTrue(br.readLine().equals("hello"));
> > +    assertTrue(br.readLine() == null);
> > +    assertEquals(3, br.getLineNumber());
> > +
> > +
> > +    br = getEBR("foo\rbaar\r\nfoo");
> > +    assertTrue(br.readLine().equals("foo"));
> > +    assertEquals('b', br.lookAhead());
> > +    assertTrue(br.readLine().equals("baar"));
> > +    assertEquals('f', br.lookAhead());
> > +    assertTrue(br.readLine().equals("foo"));
> > +    assertTrue(br.readLine() == null);
> > +  }
> > +
> > +  public void testSkip0() throws Exception {
> > +
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertEquals(0, br.skip(0));
> > +    assertEquals(0, br.skip(1));
> > +
> > +    br = getEBR("");
> > +    assertEquals(0, br.skip(1));
> > +
> > +    br = getEBR("abcdefg");
> > +    assertEquals(0, br.skip(0));
> > +    assertEquals('a', br.lookAhead());
> > +    assertEquals(0, br.skip(0));
> > +    assertEquals('a', br.lookAhead());
> > +    assertEquals(1, br.skip(1));
> > +    assertEquals('b', br.lookAhead());
> > +    assertEquals('b', br.read());
> > +    assertEquals(3, br.skip(3));
> > +    assertEquals('f', br.lookAhead());
> > +    assertEquals(2, br.skip(5));
> > +    assertTrue(br.readLine() == null);
> > +
> > +    br = getEBR("12345");
> > +    assertEquals(5, br.skip(5));
> > +    assertTrue (br.lookAhead() == ExtendedBufferedReader.END_OF_STREAM);
> > +  }
> > +
> > +  public void testSkipUntil() throws Exception {
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertEquals(0, br.skipUntil(';'));
> > +    br = getEBR("ABCDEF,GHL,,MN");
> > +    assertEquals(6, br.skipUntil(','));
> > +    assertEquals(0, br.skipUntil(','));
> > +    br.skip(1);
> > +    assertEquals(3, br.skipUntil(','));
> > +    br.skip(1);
> > +    assertEquals(0, br.skipUntil(','));
> > +    br.skip(1);
> > +    assertEquals(2, br.skipUntil(','));
> > +  }
> > +
> > +  public void testReadUntil() throws Exception {
> > +    ExtendedBufferedReader br = getEBR("");
> > +    assertTrue(br.readUntil(';').equals(""));
> > +    br = getEBR("ABCDEF;GHL;;MN");
> > +    assertTrue(br.readUntil(';').equals("ABCDEF"));
> > +    assertTrue(br.readUntil(';').length() == 0);
> > +    br.skip(1);
> > +    assertTrue(br.readUntil(';').equals("GHL"));
> > +    br.skip(1);
> > +    assertTrue(br.readUntil(';').equals(""));
> > +    br.skip(1);
> > +    assertTrue(br.readUntil(',').equals("MN"));
> > +  }
> > +
> > +  private ExtendedBufferedReader getEBR(String s) {
> > +    return new ExtendedBufferedReader(new StringReader(s));
> > +  }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigGuesserTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,88 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import java.io.ByteArrayInputStream;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * Tests for the config guesser.
> > + */
> > +public class CSVConfigGuesserTest extends TestCase {
> > +
> > +    public void testSetters() throws Exception {
> > +        CSVConfigGuesser guesser = new CSVConfigGuesser();
> > +        ByteArrayInputStream in = new ByteArrayInputStream(new byte[0]);
> > +        guesser.setInputStream(in);
> > +        assertEquals(in, guesser.getInputStream());
> > +        guesser = new CSVConfigGuesser(in);
> > +        assertEquals(in, guesser.getInputStream());
> > +        assertEquals(false, guesser.hasFieldHeader());
> > +        guesser.setHasFieldHeader(true);
> > +        assertEquals(true, guesser.hasFieldHeader());
> > +    }
> > +    /**
> > +     * Test a format like
> > +     *  1234 ; abcd ; 1234 ;
> > +     *
> > +     */
> > +    public void testConfigGuess1() {
> > +        CSVConfig expected = new CSVConfig();
> > +        expected.setDelimiter(';');
> > +        expected.setValueDelimiter(' ');
> > +        expected.setFill(CSVConfig.FILLRIGHT);
> > +        expected.setIgnoreValueDelimiter(false);
> > +        expected.setFixedWidth(true);
> > +        CSVField field = new CSVField();
> > +        field.setSize(4);
> > +        expected.addField(field);
> > +        expected.addField(field);
> > +        StringBuffer sb = new StringBuffer();
> > +        sb.append("1234;abcd;1234\n");
> > +        sb.append("abcd;1234;abcd");
> > +        ByteArrayInputStream in = new
> ByteArrayInputStream(sb.toString().getBytes());
> > +        CSVConfigGuesser guesser = new CSVConfigGuesser(in);
> > +        CSVConfig guessed = guesser.guess();
> > +        assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
> > +        assertEquals(expected.getFields().length,
> guessed.getFields().length);
> > +        assertEquals(expected.getFields()[0].getSize(),
> guessed.getFields()[0].getSize());
> > +    }
> > +    /**
> > +     * Test a format like
> > +     *  1234,123123,12312312,213123
> > +     *  1,2,3,4
> > +     *
> > +     */
> > +    public void testConfigGuess2() {
> > +        CSVConfig expected = new CSVConfig();
> > +        expected.setDelimiter(';');
> > +        expected.setValueDelimiter(' ');
> > +        expected.setFill(CSVConfig.FILLRIGHT);
> > +        expected.setIgnoreValueDelimiter(false);
> > +//        expected.setFixedWidth(false);
> > +        StringBuffer sb = new StringBuffer();
> > +        sb.append("1,2,3,4\n");
> > +        sb.append("abcd,1234,abcd,1234");
> > +        ByteArrayInputStream in = new
> ByteArrayInputStream(sb.toString().getBytes());
> > +        CSVConfigGuesser guesser = new CSVConfigGuesser(in);
> > +        CSVConfig guessed = guesser.guess();
> > +        assertEquals(expected.isFixedWidth(), guessed.isFixedWidth());
> > +    }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVConfigTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,100 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import java.util.Collection;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * Testcase for the CSVConfig
> > + */
> > +public class CSVConfigTest extends TestCase {
> > +
> > +
> > +    public void testFixedWith() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(false, config.isFixedWidth());
> > +        config.setFixedWidth(true);
> > +        assertEquals(true, config.isFixedWidth());
> > +    }
> > +
> > +    public void testFields() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(0, config.getFields().length);
> > +        config.setFields((CSVField[])null);
> > +        assertEquals(0, config.getFields().length);
> > +        config.setFields((Collection)null);
> > +        assertEquals(0, config.getFields().length);
> > +        CSVField field = new CSVField();
> > +        field.setName("field1");
> > +        config.addField(field);
> > +        assertEquals(field, config.getFields()[0]);
> > +        assertEquals(null, config.getField(null));
> > +        assertEquals(null, config.getField("field11"));
> > +        assertEquals(field, config.getField("field1"));
> > +    }
> > +
> > +    public void testFill() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(CSVConfig.FILLNONE, config.getFill());
> > +        config.setFill(CSVConfig.FILLLEFT);
> > +        assertEquals(CSVConfig.FILLLEFT, config.getFill());
> > +        config.setFill(CSVConfig.FILLRIGHT);
> > +        assertEquals(CSVConfig.FILLRIGHT, config.getFill());
> > +        assertEquals(' ', config.getFillChar());
> > +        config.setFillChar('m');
> > +        assertEquals('m', config.getFillChar());
> > +    }
> > +
> > +    public void testDelimiter() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(',', config.getDelimiter());
> > +        config.setDelimiter(';');
> > +        assertEquals(';', config.getDelimiter());
> > +        assertEquals(false, config.isDelimiterIgnored());
> > +        config.setIgnoreDelimiter(true);
> > +        assertEquals(true, config.isDelimiterIgnored());
> > +    }
> > +
> > +    public void testValueDelimiter() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals('"', config.getValueDelimiter());
> > +        config.setValueDelimiter('m');
> > +        assertEquals('m', config.getValueDelimiter());
> > +        assertEquals(true, config.isValueDelimiterIgnored());
> > +        config.setIgnoreValueDelimiter(false);
> > +        assertEquals(false, config.isValueDelimiterIgnored());
> > +    }
> > +
> > +    public void testFieldHeader() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(false, config.isFieldHeader());
> > +        config.setFieldHeader(true);
> > +        assertEquals(true, config.isFieldHeader());
> > +    }
> > +
> > +    public void testTrimEnd() {
> > +        CSVConfig config = new CSVConfig();
> > +        assertEquals(false, config.isEndTrimmed());
> > +        config.setEndTrimmed(true);
> > +        assertEquals(true, config.isEndTrimmed());
> > +    }
> > +
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVFieldTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,48 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import junit.framework.TestCase;
> > +
> > +public class CSVFieldTest extends TestCase {
> > +
> > +    public void testCSVField() {
> > +        CSVField field = new CSVField();
> > +        assertEquals(null, field.getName());
> > +        field.setName("id");
> > +        assertEquals("id", field.getName());
> > +        assertEquals(0, field.getSize());
> > +        field.setSize(10);
> > +        assertEquals(10, field.getSize());
> > +        field = new CSVField("name");
> > +        assertEquals("name", field.getName());
> > +        field = new CSVField("name", 10);
> > +        assertEquals("name", field.getName());
> > +        assertEquals(10, field.getSize());
> > +    }
> > +
> > +    public void testFill() {
> > +        CSVField field = new CSVField();
> > +        assertEquals(CSVConfig.FILLNONE, field.getFill());
> > +        assertEquals(false, field.overrideFill());
> > +        field.setFill(CSVConfig.FILLLEFT);
> > +        assertEquals(true, field.overrideFill());
> > +        assertEquals(CSVConfig.FILLLEFT, field.getFill());
> > +    }
> > +}
> >
> > Added:
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> > URL:
> http://svn.apache.org/viewvc/lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java?rev=1308225&view=auto
> >
> ==============================================================================
> > ---
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> (added)
> > +++
> lucene/dev/branches/lucene3930/solr/core/src/test/org/apache/solr/internal/csv/writer/CSVWriterTest.java
> Mon Apr  2 01:29:20 2012
> > @@ -0,0 +1,56 @@
> > +/*
> > + * Licensed to the Apache Software Foundation (ASF) under one
> > + * or more contributor license agreements.  See the NOTICE file
> > + * distributed with this work for additional information
> > + * regarding copyright ownership.  The ASF licenses this file
> > + * to you under the Apache License, Version 2.0 (the
> > + * "License"); you may not use this file except in compliance
> > + * with the License.  You may obtain a copy of the License at
> > + *
> > + * http://www.apache.org/licenses/LICENSE-2.0
> > + *
> > + * Unless required by applicable law or agreed to in writing,
> > + * software distributed under the License is distributed on an
> > + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
> > + * KIND, either express or implied.  See the License for the
> > + * specific language governing permissions and limitations
> > + * under the License.
> > + */
> > +package org.apache.solr.internal.csv.writer;
> > +
> > +import java.io.StringWriter;
> > +import java.util.HashMap;
> > +import java.util.Map;
> > +
> > +import junit.framework.TestCase;
> > +
> > +/**
> > + * The testcase for the csv writer.
> > + */
> > +public class CSVWriterTest extends TestCase {
> > +
> > +    public void testCSVConfig() {
> > +        CSVWriter writer = new CSVWriter();
> > +        assertEquals(null, writer.getConfig());
> > +        CSVConfig config = new CSVConfig();
> > +        writer.setConfig(config);
> > +        assertEquals(config, writer.getConfig());
> > +        writer = new CSVWriter(config);
> > +        assertEquals(config, writer.getConfig());
> > +    }
> > +
> > +    public void testWriter() {
> > +        CSVWriter writer = new CSVWriter();
> > +        CSVConfig config = new CSVConfig();
> > +        config.addField(new CSVField("field1", 5));
> > +        config.addField(new CSVField("field2", 4));
> > +        writer.setConfig(config);
> > +        StringWriter sw = new StringWriter();
> > +        writer.setWriter(sw);
> > +        Map map = new HashMap();
> > +        map.put("field1", "12345");
> > +        map.put("field2", "1234");
> > +        writer.writeRecord(map);
> > +        assertEquals("12345,1234\n",sw.toString());
> > +    }
> > +}
> >
> >
>
>
>
> --
> lucidimagination.com
>
> ---------------------------------------------------------------------
> To unsubscribe, e-mail: dev-unsubscribe@lucene.apache.org
> For additional commands, e-mail: dev-help@lucene.apache.org
>
>


-- 
Chris Male | Software Developer | DutchWorks | www.dutchworks.nl