You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2018/01/18 14:03:54 UTC
[08/32] sqoop git commit: SQOOP-3273: Removing com.cloudera.sqoop
packages
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestCodecMap.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/io/TestCodecMap.java b/src/test/org/apache/sqoop/io/TestCodecMap.java
new file mode 100644
index 0000000..e719218
--- /dev/null
+++ b/src/test/org/apache/sqoop/io/TestCodecMap.java
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.io;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.GzipCodec;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Rule;
+
+import org.junit.rules.ExpectedException;
+
+/**
+ * Test looking up codecs by name.
+ */
+public class TestCodecMap {
+
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private void verifyCodec(Class<?> c, String codecName)
+ throws UnsupportedCodecException {
+ CompressionCodec codec = CodecMap.getCodec(codecName, new Configuration());
+ assertEquals(codec.getClass(), c);
+ }
+
+ @Test
+ public void testGetCodecNames() {
+ // gzip is picked up from Hadoop defaults
+ assertTrue(CodecMap.getCodecNames().contains("gzip"));
+ }
+
+ @Test
+ public void testGetCodec() throws IOException {
+ verifyCodec(GzipCodec.class, "gzip");
+ verifyCodec(GzipCodec.class, "Gzip");
+ verifyCodec(GzipCodec.class, "GZIP");
+ verifyCodec(GzipCodec.class, "gzipcodec");
+ verifyCodec(GzipCodec.class, "GzipCodec");
+ verifyCodec(GzipCodec.class, "GZIPCODEC");
+ verifyCodec(GzipCodec.class, "org.apache.hadoop.io.compress.GzipCodec");
+ }
+
+ @Test
+ public void testGetShortName() throws UnsupportedCodecException {
+ verifyShortName("gzip", "org.apache.hadoop.io.compress.GzipCodec");
+ verifyShortName("default", "org.apache.hadoop.io.compress.DefaultCodec");
+
+ thrown.expect(UnsupportedCodecException.class);
+ thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException with invalid codec name during getting " +
+ "short codec name");
+ verifyShortName("NONE", "bogus");
+ }
+
+ private void verifyShortName(String expected, String codecName)
+ throws UnsupportedCodecException {
+ assertEquals(expected,
+ CodecMap.getCodecShortNameByName(codecName, new Configuration()));
+ }
+
+ @Test
+ public void testUnrecognizedCodec() throws UnsupportedCodecException {
+ thrown.expect(UnsupportedCodecException.class);
+ thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException with invalid codec name");
+ CodecMap.getCodec("bogus", new Configuration());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestLobFile.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/io/TestLobFile.java b/src/test/org/apache/sqoop/io/TestLobFile.java
new file mode 100644
index 0000000..2bc95f2
--- /dev/null
+++ b/src/test/org/apache/sqoop/io/TestLobFile.java
@@ -0,0 +1,642 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.io;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.Reader;
+import java.io.Writer;
+import java.net.URI;
+import java.nio.CharBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.CryptoOutputStream;
+import org.apache.hadoop.crypto.JceAesCtrCryptoCodec;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.junit.Before;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Answers.CALLS_REAL_METHODS;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.withSettings;
+
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Test the LobFile reader/writer implementation.
+ */
+public class TestLobFile {
+
+ public static final Log LOG = LogFactory.getLog(
+ TestLobFile.class.getName());
+
+ public static final Path TEMP_BASE_DIR;
+
+ static {
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ if (!tmpDir.endsWith(File.separator)) {
+ tmpDir = tmpDir + File.separator;
+ }
+
+ TEMP_BASE_DIR = new Path(new Path(tmpDir), "lobtest");
+ }
+
+ private Configuration conf;
+ private FileSystem fs;
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.set("fs.default.name", "file:///");
+
+ fs = FileSystem.getLocal(conf);
+ fs.mkdirs(TEMP_BASE_DIR);
+ }
+
+ private long[] writeClobFile(Path p, String codec,
+ String... records) throws Exception {
+ if (fs.exists(p)) {
+ fs.delete(p, false);
+ }
+
+ // memorize the offsets of each record we write.
+ long[] offsets = new long[records.length];
+
+ // Create files with four entries per index segment.
+ LobFile.Writer writer = LobFile.create(p, conf, true, codec, 4);
+
+ int i = 0;
+ for (String r : records) {
+ offsets[i++] = writer.tell();
+ Writer w = writer.writeClobRecord(r.length());
+ w.write(r);
+ w.close();
+ }
+
+ writer.close();
+ return offsets;
+ }
+
+ private void verifyClobFile(Path p, String... expectedRecords)
+ throws Exception {
+
+ LobFile.Reader reader = LobFile.open(p, conf);
+
+ int recNum = 0;
+
+ while (reader.next()) {
+ // We should have a record of the same length as the expected one.
+ String expected = expectedRecords[recNum];
+ assertTrue(reader.isRecordAvailable());
+ assertEquals(expected.length(), reader.getRecordLen());
+ Reader r = reader.readClobRecord();
+
+ // Read in the record and assert that we got enough characters out.
+ CharBuffer buf = CharBuffer.allocate(expected.length());
+ int bytesRead = 0;
+ while (bytesRead < expected.length()) {
+ int thisRead = r.read(buf);
+ LOG.info("performed read of " + thisRead + " chars");
+ if (-1 == thisRead) {
+ break;
+ }
+
+ bytesRead += thisRead;
+ }
+
+ LOG.info("Got record of " + bytesRead + " chars");
+ assertEquals(expected.length(), bytesRead);
+ char[] charData = buf.array();
+ String finalRecord = new String(charData);
+ assertEquals(expected, finalRecord);
+
+ recNum++;
+ }
+
+ // Check that we got everything.
+ assertEquals(expectedRecords.length, recNum);
+
+ reader.close();
+
+ thrown.expect(IOException.class);
+ thrown.reportMissingExceptionWithMessage("Expected IOException calling next after close");
+ reader.next();
+
+ // A second close shouldn't hurt anything. This should be a no-op.
+ reader.close();
+ }
+
+ private void runClobFileTest(Path p, String codec,
+ String... records) throws Exception {
+ writeClobFile(p, codec, records);
+ verifyClobFile(p, records);
+ fs.delete(p, false);
+ }
+
+ @Test
+ public void testEmptyRecord() throws Exception {
+ runClobFileTest(new Path(TEMP_BASE_DIR, "empty.lob"), null);
+ }
+
+ @Test
+ public void testSingleRecord() throws Exception {
+ runClobFileTest(new Path(TEMP_BASE_DIR, "single.lob"),
+ null, "this is a single record!");
+ }
+
+ @Test
+ public void testMultiRecords() throws Exception {
+ runClobFileTest(new Path(TEMP_BASE_DIR, "multi.lob"),
+ CodecMap.NONE,
+ "this is the first record",
+ "this is the second record. I assure you that this record is long.",
+ "yet one more record graces this file.");
+ }
+
+ @Test
+ public void testMultiIndexSegments() throws Exception {
+ // Test that we can use multiple IndexSegments.
+ runClobFileTest(new Path(TEMP_BASE_DIR, "multi-index.lob"),
+ null,
+ "this is the first record",
+ "this is the second record. I assure you that this record is long.",
+ "record number three",
+ "last one in first index segment",
+ "first in the second index segment",
+ "yet one more record graces this file.");
+ }
+
+ /**
+ * Run a test where we read only a fraction of the first record,
+ * but then read the second record completely. Verify that we
+ * can re-align on a record boundary correctly. This test requires
+ * at least 3 records.
+ *
+ * @param p the path to the file to create.
+ * @param firstLine the first line of the first reord
+ * @param records All of the records to write to the file.
+ */
+ private void runLineAndRecordTest(Path p, String firstLine,
+ String... records) throws Exception {
+
+ assertTrue("This test requires 3+ records", records.length > 2);
+
+ writeClobFile(p, null, records);
+
+ LobFile.Reader reader = LobFile.open(p, conf);
+
+ // We should not yet be aligned.
+ assertFalse(reader.isRecordAvailable());
+ assertTrue(reader.next());
+ // Now we should be.
+ assertTrue(reader.isRecordAvailable());
+
+ // Read just one line from the record.
+ Reader r = reader.readClobRecord();
+ BufferedReader br = new BufferedReader(r);
+ String line = br.readLine();
+ assertEquals(firstLine, line);
+
+ br.close();
+ r.close();
+
+ // We should no longer be aligned on a record start.
+ assertFalse(reader.isRecordAvailable());
+
+ // We should now be able to get to record two.
+ assertTrue(reader.next());
+
+ // This should be nicely aligned even if the first record was not
+ // completely consumed by a client.
+ r = reader.readClobRecord();
+ CharBuffer buf = CharBuffer.allocate(records[1].length());
+ r.read(buf);
+ r.close();
+ char[] chars = buf.array();
+ String s = new String(chars);
+ assertEquals(records[1], s);
+
+ // Close the reader before we consume the entire file.
+ reader.close();
+ assertFalse(reader.isRecordAvailable());
+ }
+
+ @Test
+ public void testVeryShortRead() throws Exception {
+ // Read only a small fraction of a record, ensure that we can
+ // read the next record, even when we've left more than a 16-byte
+ // quantity in the readahead buffer.
+
+ Path p = new Path(TEMP_BASE_DIR, "shortread.lob");
+ final String FIRST_LINE = "line1";
+ final String SECOND_LINE =
+ "This contains much more in the record than just one line.";
+ final String RECORD2 = "here is the second record.";
+ final String RECORD3 = "The 3rd record, which we won't actually read.";
+
+ runLineAndRecordTest(p, FIRST_LINE,
+ FIRST_LINE + "\n" + SECOND_LINE,
+ RECORD2,
+ RECORD3);
+
+ }
+
+ @Test
+ public void testIncompleteOverread() throws Exception {
+ // Read most of the first record so that we partially consume the
+ // next record start mark; make sure we realign properly.
+
+ Path p = new Path(TEMP_BASE_DIR, "longread.lob");
+ final String FIRST_LINE = "this is a really long line of text to read!";
+ final String SECOND_LINE = "not this.";
+ final String RECORD2 = "Here is yet another record to verify.";
+ final String RECORD3 = "Nobody cares about record 3.";
+
+ runLineAndRecordTest(p, FIRST_LINE,
+ FIRST_LINE + "\n" + SECOND_LINE,
+ RECORD2,
+ RECORD3);
+ }
+
+ @Test
+ public void testSeekToRecord() throws Exception {
+ // Seek past the first two records and read the third.
+
+ Path p = new Path(TEMP_BASE_DIR, "seek.lob");
+ String[] records = {
+ "this is the first record!",
+ "here comes record number two. It is a bit longer.",
+ "this is the third record. we can read it.",
+ };
+
+ // Write the file and memorize when the third record starts.
+ LobFile.Writer writer = LobFile.create(p, conf, true);
+
+ int recNum = 0;
+ long rec3Start = 0;
+ for (String r : records) {
+ Writer w = writer.writeClobRecord(r.length());
+ w.write(r);
+ w.close();
+ writer.finishRecord();
+ if (recNum == 1) {
+ rec3Start = writer.tell();
+ LOG.info("Record three start: " + rec3Start);
+ }
+ recNum++;
+ }
+
+ writer.close();
+
+ // Now reopen the file for read, seek to the third record, and get it.
+ LobFile.Reader reader = LobFile.open(p, conf);
+ reader.seek(rec3Start);
+ assertTrue(reader.next());
+ assertTrue(reader.isRecordAvailable());
+ assertEquals(rec3Start, reader.getRecordOffset());
+
+ Reader r = reader.readClobRecord();
+ CharBuffer buf = CharBuffer.allocate(records[2].length());
+ r.read(buf);
+ r.close();
+ char[] chars = buf.array();
+ String s = new String(chars);
+ assertEquals(records[2], s);
+
+ r.close();
+ reader.close();
+ }
+
+
+ /**
+ * Verifies that the next record in the LobFile is the expected one.
+ */
+ private void verifyNextRecord(LobFile.Reader reader, long expectedId,
+ String expectedRecord) throws Exception {
+ assertTrue(reader.next());
+ assertTrue(reader.isRecordAvailable());
+ assertEquals(expectedId, reader.getRecordId());
+
+ Reader r = reader.readClobRecord();
+ CharBuffer buf = CharBuffer.allocate(expectedRecord.length());
+ int bytesRead = 0;
+ while (bytesRead < expectedRecord.length()) {
+ int thisRead = r.read(buf);
+ if (-1 == thisRead) {
+ break;
+ }
+
+ bytesRead += thisRead;
+ }
+
+ LOG.info("Got record of " + bytesRead + " chars");
+ assertEquals(expectedRecord.length(), bytesRead);
+
+ char[] charData = buf.array();
+ String finalRecord = new String(charData);
+ assertEquals(expectedRecord, finalRecord);
+ }
+
+ @Test
+ public void testManySeeks() throws Exception {
+ // Test that we can do gymnastics with seeking between records.
+
+ Path p = new Path(TEMP_BASE_DIR, "manyseeks.lob");
+
+ String[] records = {
+ "first record",
+ "second record",
+ "the third record",
+ "rec4 is the last in IndexSeg 0",
+ "rec5 is first in IndexSeg 1",
+ "rec6 is yet another record",
+ "rec7 is starting to feel boring",
+ "rec8 is at the end of seg 1",
+ "rec9 is all by itself in seg 2",
+ };
+
+ // Write the records to a file, save their offsets.
+ long[] offsets = writeClobFile(p, null, records);
+
+ // Sanity check that we can stream the file.
+ verifyClobFile(p, records);
+
+ // Open a handle to the file.
+ LobFile.Reader reader = LobFile.open(p, conf);
+
+ // Seeking to offset 0 should return the first record.
+ reader.seek(0);
+ verifyNextRecord(reader, 0, records[0]);
+
+ // Seek to the last item in the first IndexSegment.
+ reader.seek(offsets[3]);
+ verifyNextRecord(reader, 3, records[3]);
+
+ // Seek to just ahead of that same record.
+ reader.seek(offsets[3] - 10);
+ verifyNextRecord(reader, 3, records[3]);
+
+ // Seek (backwards) to the first record.
+ reader.seek(offsets[0]);
+ verifyNextRecord(reader, 0, records[0]);
+
+ // Seek to first record in second IndexSegment.
+ reader.seek(offsets[4]);
+ verifyNextRecord(reader, 4, records[4]);
+
+ // Move backwards.
+ reader.seek(0);
+
+ // Seek to "no man's land" between last offset in first IndexSeg
+ // and the first offset in second IndexSegment. Result should be
+ // the first record in second InexSegment.
+ reader.seek(offsets[4] - 10);
+ verifyNextRecord(reader, 4, records[4]);
+
+ // Seek to past the last record. No record should be returned.
+ reader.seek(offsets[8] + 4);
+ assertFalse("Found a record past last record start.", reader.next());
+
+ // Seek to somewhere in the middle of IndexSegment 0.
+ // This should recover just fine.
+ reader.seek(offsets[2]);
+ verifyNextRecord(reader, 2, records[2]);
+
+ // Seek to last record in IndexSegment 1.
+ reader.seek(offsets[3] - 1);
+ verifyNextRecord(reader, 3, records[3]);
+
+ // And make sure that iteration picks up naturally from there.
+ verifyNextRecord(reader, 4, records[4]);
+
+ // Seek well past the end of the file. No record should be returned.
+ reader.seek(50000000);
+ assertFalse("Found a record past expected end-of-file", reader.next());
+
+ // Seek to somewhere in the index.
+ reader.seek(offsets[8] + 32);
+ assertFalse("Found a record past beginning of index", reader.next());
+
+ // Seek to the last record (exact hit). This is a singleton IndexSegment.
+ reader.seek(offsets[8]);
+ verifyNextRecord(reader, 8, records[8]);
+
+ // Seek to no-man's-land ahead of last record.
+ reader.seek(offsets[8] - 3);
+ verifyNextRecord(reader, 8, records[8]);
+
+ reader.close();
+ }
+
+ /**
+ * Verifies that a record to be read from a lob file has
+ * as many bytes as we expect, and that the bytes are what we
+ * expect them to be. Assumes that the bytes are such that
+ * input[i] == i + offset.
+ *
+ * @param reader the LobFile reader to consume data from
+ * @param expectedDeclaredLen the size we expect the LobFile to declare
+ * its record length as.
+ * @param expectedActualLen the true number of bytes we expect to read in
+ * the record.
+ * @param offset the offset amount for each of the elements of the array.
+ */
+ private void verifyBlobRecord(LobFile.Reader reader,
+ long expectedDeclaredLen, long expectedActualLen,
+ int offset) throws Exception {
+
+ assertTrue(reader.next());
+ assertTrue(reader.isRecordAvailable());
+ assertEquals(expectedDeclaredLen, reader.getRecordLen());
+
+ InputStream is = reader.readBlobRecord();
+
+ byte[] bytes = new byte[(int) expectedActualLen];
+ int numRead = is.read(bytes);
+ assertEquals(expectedActualLen, numRead);
+
+ for (int i = 0; i < numRead; i++) {
+ assertEquals(i + offset, (int) bytes[i]);
+ }
+
+ is.close();
+ }
+
+ /**
+ * Write a binary record to a LobFile. This allows the declared length
+ * of the record to disagree with the actual length (the actual length
+ * should be >= the declared length).
+ * The record written will have values v[i] = i + offset.
+ *
+ * @param writer the LobFile writer to put the record into
+ * @param declaredLen the length value written into the file itself
+ * @param actualLen the true number of bytes to write
+ * @param offset an amount to adjust each record's byte values by.
+ */
+ private void writeBlobRecord(LobFile.Writer writer, long declaredLen,
+ long actualLen, int offset) throws Exception {
+ OutputStream os = writer.writeBlobRecord(declaredLen);
+ for (int i = 0; i < actualLen; i++) {
+ os.write(i + offset);
+ }
+
+ os.close();
+ writer.finishRecord();
+ }
+
+ /**
+ * Verifies a number of records that all have the same declared
+ * and actual record lengths.
+ *
+ * @param p the path to the LobFile to open
+ * @param numRecords the number of records to expect
+ * @param declaredLen the declared length of each record in the file
+ * @param actualLen the true number of bytes we expect to read per record.
+ */
+ private void verifyBlobRecords(Path p, int numRecords,
+ long declaredLen, long actualLen) throws Exception {
+
+ LobFile.Reader reader = LobFile.open(p, conf);
+ for (int i = 0; i < numRecords; i++) {
+ verifyBlobRecord(reader, declaredLen, actualLen, i);
+ }
+ assertFalse(reader.next());
+ reader.close();
+ }
+
+ @Test
+ public void testBinaryRecords() throws Exception {
+ // Write a BLOB file and read it all back.
+
+ final long RECORD_LEN = 32;
+ final int NUM_RECORDS = 2;
+ Path p = new Path(TEMP_BASE_DIR, "binary.lob");
+ LobFile.Writer writer = LobFile.create(p, conf);
+
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ writeBlobRecord(writer, RECORD_LEN, RECORD_LEN, i);
+ }
+
+ writer.close();
+
+ // Now check the read-back on those records.
+ verifyBlobRecords(p, NUM_RECORDS, RECORD_LEN, RECORD_LEN);
+ }
+
+ @Test
+ public void testOverLengthBinaryRecord() throws Exception {
+ // Write a record with a declared length shorter than the
+ // actual length, and read it back.
+
+ final long ACTUAL_RECORD_LEN = 48;
+ final long DECLARED_RECORD_LEN = 32;
+ final int NUM_RECORDS = 2;
+
+ Path p = new Path(TEMP_BASE_DIR, "overlength.lob");
+ LobFile.Writer writer = LobFile.create(p, conf);
+
+ for (int i = 0; i < NUM_RECORDS; i++) {
+ writeBlobRecord(writer, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN, i);
+ }
+
+ writer.close();
+
+ // Now read them back.
+ verifyBlobRecords(p, NUM_RECORDS, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN);
+ }
+
+ private void runCompressedTest(String codec) throws Exception {
+ LOG.info("Testing with codec: " + codec);
+ Path p = new Path(TEMP_BASE_DIR, "compressed-" + codec + ".lob");
+ String[] records = {
+ "this is the first record, It should be compressed a lot!",
+ "record 2 record 2 record 2 record 2 2 2 2 2 2 2 2 2 2 2 2",
+ "and a third and a third yes this is the third",
+ };
+
+ runClobFileTest(p, codec, records);
+ }
+
+ @Test
+ public void testCompressedFile() throws Exception {
+ // Test all the various compression codecs.
+
+ // The following values for 'codec' should pass.
+ runCompressedTest(null);
+ runCompressedTest(CodecMap.NONE);
+ runCompressedTest(CodecMap.DEFLATE);
+
+ thrown.expect(UnsupportedCodecException.class);
+ thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException for lzo");
+ runCompressedTest(CodecMap.LZO);
+ }
+
+ @Test
+ public void testCryptoOutputStreamClosingDoesNotThrowExceptionAndClosedProperly() throws Exception {
+ // Tests that closing CryptoOutputStream doesn't throw exception neither with Java 7 nor with Java 8
+ // For a detailed explanation see SQOOP-3243
+ CryptoOutputStream cryptoOutputStream = createCryptoOutputStream();
+ FSDataOutputStream wrappedCryptoOutputStream = new FSDataOutputStream(cryptoOutputStream, null);
+
+ Path mockPath = spy(new Path("file://" + TEMP_BASE_DIR, "binary.lob"));
+ FileSystem mockFileSystem = mock(FileSystem.class, withSettings().defaultAnswer(CALLS_REAL_METHODS.get()));
+
+ doReturn(mockFileSystem).when(mockPath).getFileSystem(conf);
+ doReturn(null).when(mockFileSystem).getWorkingDirectory();
+ doReturn(wrappedCryptoOutputStream).when(mockFileSystem).create(mockPath);
+ doReturn(new URI("file:///")).when(mockFileSystem).getUri();
+
+ LobFile.Writer writer = LobFile.create(mockPath, conf);
+
+ writer.close();
+
+ verify(cryptoOutputStream).close();
+ }
+
+
+ public CryptoOutputStream createCryptoOutputStream() throws Exception {
+ final byte[] BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
+ Path p = new Path(TEMP_BASE_DIR, "binary.lob");
+
+ FSDataOutputStream fsDataOutputStream = fs.create(p);
+ CryptoOutputStream cryptoOutputStream = spy(new CryptoOutputStream(fsDataOutputStream, new JceAesCtrCryptoCodec(), 512, BYTES, BYTES));
+
+ return cryptoOutputStream;
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestNamedFifo.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/io/TestNamedFifo.java b/src/test/org/apache/sqoop/io/TestNamedFifo.java
new file mode 100644
index 0000000..a93784e
--- /dev/null
+++ b/src/test/org/apache/sqoop/io/TestNamedFifo.java
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.io;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+
+import org.apache.hadoop.util.StringUtils;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Shell;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+/**
+ * Test the named fifo utility.
+ */
+public class TestNamedFifo {
+
+ public static final Log LOG = LogFactory.getLog(
+ TestNamedFifo.class.getName());
+
+ public static final Path TEMP_BASE_DIR;
+
+ static {
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ if (!tmpDir.endsWith(File.separator)) {
+ tmpDir = tmpDir + File.separator;
+ }
+
+ TEMP_BASE_DIR = new Path(new Path(tmpDir), "namedfifo");
+ }
+
+ private Configuration conf;
+ private FileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.set("fs.default.name", "file:///");
+
+ fs = FileSystem.getLocal(conf);
+ fs.mkdirs(TEMP_BASE_DIR);
+ }
+
+ static final String MSG = "THIS IS THE MESSAGE\n";
+ static final String MSG2 = "Here is a follow-up.\n";
+
+ private static class ReaderThread extends Thread {
+ private File file;
+ private IOException exception;
+
+ public ReaderThread(File f) {
+ this.file = f;
+ }
+
+ /** return any exception during the run method. */
+ public IOException getException() {
+ return this.exception;
+ }
+
+ public void run() {
+ BufferedReader r = null;
+ try {
+ r = new BufferedReader(new InputStreamReader(
+ new FileInputStream(file)));
+
+ // Assert that after a flush, we get back what we wrote.
+ String line = r.readLine();
+ if (!MSG.trim().equals(line)) {
+ throw new IOException("Expected " + MSG.trim() + " but got "
+ + line);
+ }
+
+ // Assert that after closing the writer, we get back what
+ // we wrote again.
+ line = r.readLine();
+ if (null == line) {
+ throw new IOException("line2 was null");
+ } else if (!MSG2.trim().equals(line)) {
+ throw new IOException("Expected " + MSG2.trim() + " but got "
+ + line);
+ }
+ } catch (IOException ioe) {
+ this.exception = ioe;
+ } finally {
+ if (null != r) {
+ try {
+ r.close();
+ } catch (IOException ioe) {
+ LOG.warn("Error closing reader: " + ioe);
+ }
+ }
+ }
+ }
+ }
+
+ private static class WriterThread extends Thread {
+ private File file;
+ private IOException exception;
+
+ public WriterThread(File f) {
+ this.file = f;
+ }
+
+ /** return any exception during the run method. */
+ public IOException getException() {
+ return this.exception;
+ }
+
+ public void run() {
+ BufferedWriter w = null;
+ try {
+ w = new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(file)));
+
+ w.write(MSG);
+ w.flush();
+
+ w.write(MSG2);
+ } catch (IOException ioe) {
+ this.exception = ioe;
+ } finally {
+ if (null != w) {
+ try {
+ w.close();
+ } catch (IOException ioe) {
+ LOG.warn("Error closing writer: " + ioe);
+ }
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testNamedFifo() throws Exception {
+
+ if (Shell.WINDOWS) {
+ // NamedFifo uses Linux specific commands like mknod
+ // and mkfifo, so skip the test on Windows OS
+ LOG.warn("Named FIFO is not supported on Windows. Skipping test");
+ return;
+ }
+
+ File root = new File(TEMP_BASE_DIR.toString());
+ File fifo = new File(root, "foo-fifo");
+
+ NamedFifo nf = new NamedFifo(fifo);
+ nf.create();
+
+ File returned = nf.getFile();
+
+ // These should be the same object.
+ assertEquals(fifo, returned);
+
+ ReaderThread rt = new ReaderThread(returned);
+ WriterThread wt = new WriterThread(returned);
+
+ rt.start();
+ wt.start();
+
+ rt.join();
+ wt.join();
+
+ IOException rex = rt.getException();
+ IOException wex = wt.getException();
+
+ if (null != rex) {
+ LOG.error("reader exception: " + StringUtils.stringifyException(rex));
+ }
+
+ if (null != wex) {
+ LOG.error("writer exception: " + StringUtils.stringifyException(wex));
+ }
+
+ assertNull(rex);
+ assertNull(wex);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java b/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java
new file mode 100644
index 0000000..c59aa26
--- /dev/null
+++ b/src/test/org/apache/sqoop/io/TestSplittableBufferedWriter.java
@@ -0,0 +1,318 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.io;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.compress.GzipCodec;
+
+import org.apache.sqoop.testutil.ImportJobTestCase;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * Test that the splittable buffered writer system works.
+ */
+public class TestSplittableBufferedWriter {
+
+ public static final Log LOG = LogFactory.getLog(
+ TestSplittableBufferedWriter.class.getName());
+
+ private String getWriteDir() {
+ return new File(ImportJobTestCase.TEMP_BASE_DIR,
+ "bufferedWriterTest").toString();
+ }
+
+ private Path getWritePath() {
+ return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest");
+ }
+
+ /** Create the directory where we'll write our test files to; and
+ * make sure it has no files in it.
+ */
+ private void ensureEmptyWriteDir() throws IOException {
+ FileSystem fs = FileSystem.getLocal(getConf());
+ Path writeDir = getWritePath();
+
+ fs.mkdirs(writeDir);
+
+ FileStatus [] stats = fs.listStatus(writeDir);
+
+ for (FileStatus stat : stats) {
+ if (stat.isDir()) {
+ fail("setUp(): Write directory " + writeDir
+ + " contains subdirectories");
+ }
+
+ LOG.debug("setUp(): Removing " + stat.getPath());
+ if (!fs.delete(stat.getPath(), false)) {
+ fail("setUp(): Could not delete residual file " + stat.getPath());
+ }
+ }
+
+ if (!fs.exists(writeDir)) {
+ fail("setUp: Could not create " + writeDir);
+ }
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ ensureEmptyWriteDir();
+ }
+
+ private Configuration getConf() {
+ Configuration conf = new Configuration();
+ conf.set("fs.default.name", "file:///");
+ return conf;
+ }
+
+ /** Verifies contents of an InputStream. Closes the InputStream on
+ * its way out. Fails the test if the file doesn't match the expected set
+ * of lines.
+ */
+ private void verifyFileContents(InputStream is, String [] lines)
+ throws IOException {
+ BufferedReader r = new BufferedReader(new InputStreamReader(is));
+ try {
+ for (String expectedLine : lines) {
+ String actualLine = r.readLine();
+ assertNotNull(actualLine);
+ assertEquals("Input line mismatch", expectedLine, actualLine);
+ }
+
+ assertNull("Stream had additional contents after expected line",
+ r.readLine());
+ } finally {
+ r.close();
+ try {
+ is.close();
+ } catch (IOException ioe) {
+ // ignore IOE; may be closed by reader.
+ }
+ }
+ }
+
+ private void verifyFileExists(Path p) throws IOException {
+ FileSystem fs = FileSystem.getLocal(getConf());
+ assertTrue("File not found: " + p, fs.exists(p));
+ }
+
+ private void verifyFileDoesNotExist(Path p) throws IOException {
+ FileSystem fs = FileSystem.getLocal(getConf());
+ assertFalse("File found: " + p + " and we did not expect it", fs.exists(p));
+ }
+
+ @Test
+ public void testNonSplittingTextFile() throws IOException {
+ SplittingOutputStream os = new SplittingOutputStream(getConf(),
+ getWritePath(), "nonsplit-", 0, null);
+ try {
+ SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
+ try {
+ w.allowSplit();
+ w.write("This is a string!");
+ w.newLine();
+ w.write("This is another string!");
+ w.allowSplit();
+ } finally {
+ w.close();
+ }
+ } finally {
+ try {
+ os.close();
+ } catch (IOException ioe) {
+ // Ignored; may be thrown because w is already closed.
+ }
+ }
+
+ // Ensure we made exactly one file.
+ Path writePath = new Path(getWritePath(), "nonsplit-00000");
+ Path badPath = new Path(getWritePath(), "nonsplit-00001");
+ verifyFileExists(writePath);
+ verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file.
+
+ // Now ensure all the data got there.
+ String [] expectedLines = {
+ "This is a string!",
+ "This is another string!",
+ };
+
+ InputStream fis = new FileInputStream(new File(getWriteDir(),
+ "nonsplit-00000"));
+ try {
+ verifyFileContents(fis, expectedLines);
+ } finally {
+ try {
+ fis.close();
+ } catch (IOException ioe) {
+ // Ignored; may be closed by verifyFileContents().
+ }
+ }
+ }
+
+ @Test
+ public void testNonSplittingGzipFile() throws IOException {
+ SplittingOutputStream os = new SplittingOutputStream(getConf(),
+ getWritePath(), "nonsplit-", 0, new GzipCodec());
+ SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
+ try {
+ w.allowSplit();
+ w.write("This is a string!");
+ w.newLine();
+ w.write("This is another string!");
+ w.allowSplit();
+ } finally {
+ w.close();
+ }
+
+ // Ensure we made exactly one file.
+ Path writePath = new Path(getWritePath(), "nonsplit-00000.gz");
+ Path badPath = new Path(getWritePath(), "nonsplit-00001.gz");
+ verifyFileExists(writePath);
+ verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file.
+
+ // Now ensure all the data got there.
+ String [] expectedLines = {
+ "This is a string!",
+ "This is another string!",
+ };
+ verifyFileContents(
+ new GZIPInputStream(new FileInputStream(new File(getWriteDir(),
+ "nonsplit-00000.gz"))), expectedLines);
+ }
+
+ @Test
+ public void testSplittingTextFile() throws IOException {
+ SplittingOutputStream os = new SplittingOutputStream(getConf(),
+ getWritePath(), "split-", 10, null);
+ try {
+ SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
+ try {
+ w.allowSplit();
+ w.write("This is a string!");
+ w.newLine();
+ w.write("This is another string!");
+ } finally {
+ w.close();
+ }
+ } finally {
+ try {
+ os.close();
+ } catch (IOException ioe) {
+ // Ignored; may be thrown because w is already closed.
+ }
+ }
+
+ // Ensure we made exactly two files.
+ Path writePath = new Path(getWritePath(), "split-00000");
+ Path writePath2 = new Path(getWritePath(), "split-00001");
+ Path badPath = new Path(getWritePath(), "split-00002");
+ verifyFileExists(writePath);
+ verifyFileExists(writePath2);
+ verifyFileDoesNotExist(badPath); // Ensure we didn't make three files.
+
+ // Now ensure all the data got there.
+ String [] expectedLines0 = {
+ "This is a string!",
+ };
+ InputStream fis = new FileInputStream(new File(getWriteDir(),
+ "split-00000"));
+ try {
+ verifyFileContents(fis, expectedLines0);
+ } finally {
+ try {
+ fis.close();
+ } catch (IOException ioe) {
+ // ignored; may be generated because fis closed in verifyFileContents.
+ }
+ }
+
+ String [] expectedLines1 = {
+ "This is another string!",
+ };
+ fis = new FileInputStream(new File(getWriteDir(), "split-00001"));
+ try {
+ verifyFileContents(fis, expectedLines1);
+ } finally {
+ try {
+ fis.close();
+ } catch (IOException ioe) {
+ // Ignored; may be thrown because it's closed in verifyFileContents.
+ }
+ }
+ }
+
+ @Test
+ public void testSplittingGzipFile() throws IOException {
+ SplittingOutputStream os = new SplittingOutputStream(getConf(),
+ getWritePath(), "splitz-", 3, new GzipCodec());
+ SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
+ try {
+ w.write("This is a string!");
+ w.newLine();
+ w.write("This is another string!");
+ } finally {
+ w.close();
+ }
+
+ // Ensure we made exactly two files.
+ Path writePath = new Path(getWritePath(), "splitz-00000.gz");
+ Path writePath2 = new Path(getWritePath(), "splitz-00001.gz");
+ Path badPath = new Path(getWritePath(), "splitz-00002.gz");
+ verifyFileExists(writePath);
+ verifyFileExists(writePath2);
+ verifyFileDoesNotExist(badPath); // Ensure we didn't make three files.
+
+ // Now ensure all the data got there.
+ String [] expectedLines0 = {
+ "This is a string!",
+ };
+ verifyFileContents(
+ new GZIPInputStream(new FileInputStream(new File(getWriteDir(),
+ "splitz-00000.gz"))), expectedLines0);
+
+ String [] expectedLines1 = {
+ "This is another string!",
+ };
+ verifyFileContents(
+ new GZIPInputStream(new FileInputStream(new File(getWriteDir(),
+ "splitz-00001.gz"))), expectedLines1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestBlobRef.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/lib/TestBlobRef.java b/src/test/org/apache/sqoop/lib/TestBlobRef.java
new file mode 100644
index 0000000..b271d3c
--- /dev/null
+++ b/src/test/org/apache/sqoop/lib/TestBlobRef.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.lib;
+
+import java.io.*;
+
+import org.apache.sqoop.testutil.BaseSqoopTestCase;
+import org.apache.sqoop.testutil.CommonArgs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.io.LobFile;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test that the BlobRef.parse() method does the right thing.
+ * Note that we don't support inline parsing here; we only expect this to
+ * really work for external BLOBs.
+ */
+public class TestBlobRef {
+
+ @Test
+ public void testEmptyStr() {
+ BlobRef r = BlobRef.parse("");
+ assertFalse(r.isExternal());
+ }
+
+ @Test
+ public void testInline() throws IOException {
+ BlobRef r = BlobRef.parse("foo");
+ assertFalse(r.isExternal());
+ }
+
+ @Test
+ public void testEmptyFile() {
+ BlobRef r = BlobRef.parse("externalLob()");
+ assertFalse(r.isExternal());
+
+ r = BlobRef.parse("externalLob(lf,,0,0)");
+ assertTrue(r.isExternal());
+ assertEquals("externalLob(lf,,0,0)", r.toString());
+ }
+
+ @Test
+ public void testInlineNearMatch() {
+ BlobRef r = BlobRef.parse("externalLob(foo)bar");
+ assertFalse(r.isExternal());
+
+ r = BlobRef.parse("externalLob(foo)");
+ assertFalse(r.isExternal());
+
+ r = BlobRef.parse("externalLob(lf,foo)");
+ assertFalse(r.isExternal());
+
+ r = BlobRef.parse("externalLob(lf,foo,1,2)x");
+ assertFalse(r.isExternal());
+ }
+
+ @Test
+ public void testExternal() throws IOException {
+ final byte [] DATA = { 1, 2, 3, 4, 5 };
+ final String FILENAME = "blobdata";
+
+ doExternalTest(DATA, FILENAME);
+ }
+
+ @Test
+ public void testExternalSubdir() throws IOException {
+ final byte [] DATA = { 1, 2, 3, 4, 5 };
+ final String FILENAME = "_lob/blobdata";
+
+ try {
+ doExternalTest(DATA, FILENAME);
+ } finally {
+ // remove dir we made.
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ Path lobDir = new Path(new Path(tmpDir), "_lob");
+ fs.delete(lobDir, true);
+ }
+ }
+
+ private void doExternalTest(final byte [] data, final String filename)
+ throws IOException {
+
+ Configuration conf = new Configuration();
+ if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
+ conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
+ }
+ FileSystem fs = FileSystem.get(conf);
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+
+ Path tmpPath = new Path(tmpDir);
+ Path blobFile = new Path(tmpPath, filename);
+
+ // make any necessary parent dirs.
+ Path blobParent = blobFile.getParent();
+ if (!fs.exists(blobParent)) {
+ fs.mkdirs(blobParent);
+ }
+
+ LobFile.Writer lw = LobFile.create(blobFile, conf, false);
+ try {
+ long off = lw.tell();
+ long len = data.length;
+ OutputStream os = lw.writeBlobRecord(len);
+ os.write(data, 0, data.length);
+ os.close();
+ lw.close();
+
+ String refString = "externalLob(lf," + filename
+ + "," + off + "," + len + ")";
+ BlobRef blob = BlobRef.parse(refString);
+ assertTrue(blob.isExternal());
+ assertEquals(refString, blob.toString());
+ InputStream is = blob.getDataStream(conf, tmpPath);
+ assertNotNull(is);
+
+ byte [] buf = new byte[4096];
+ int bytes = is.read(buf, 0, 4096);
+ is.close();
+
+ assertEquals(data.length, bytes);
+ for (int i = 0; i < bytes; i++) {
+ assertEquals(data[i], buf[i]);
+ }
+ } finally {
+ fs.delete(blobFile, false);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestBooleanParser.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/lib/TestBooleanParser.java b/src/test/org/apache/sqoop/lib/TestBooleanParser.java
new file mode 100644
index 0000000..914ab37
--- /dev/null
+++ b/src/test/org/apache/sqoop/lib/TestBooleanParser.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.lib;
+
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test the boolean parser.
+ */
+public class TestBooleanParser {
+
+ @Test
+ public void testBoolParser() {
+ assertTrue(BooleanParser.valueOf("true"));
+ assertTrue(BooleanParser.valueOf("TRUE"));
+ assertTrue(BooleanParser.valueOf("True"));
+ assertTrue(BooleanParser.valueOf("t"));
+ assertTrue(BooleanParser.valueOf("T"));
+ assertTrue(BooleanParser.valueOf("on"));
+ assertTrue(BooleanParser.valueOf("On"));
+ assertTrue(BooleanParser.valueOf("ON"));
+ assertTrue(BooleanParser.valueOf("yes"));
+ assertTrue(BooleanParser.valueOf("yEs"));
+ assertTrue(BooleanParser.valueOf("YES"));
+ assertTrue(BooleanParser.valueOf("1"));
+
+ assertFalse(BooleanParser.valueOf(null));
+
+ assertFalse(BooleanParser.valueOf("no"));
+ assertFalse(BooleanParser.valueOf("false"));
+ assertFalse(BooleanParser.valueOf("FALSE"));
+ assertFalse(BooleanParser.valueOf("0"));
+ assertFalse(BooleanParser.valueOf("off"));
+ assertFalse(BooleanParser.valueOf("OFF"));
+ assertFalse(BooleanParser.valueOf("anything else in the world"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestClobRef.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/lib/TestClobRef.java b/src/test/org/apache/sqoop/lib/TestClobRef.java
new file mode 100644
index 0000000..f94d1a8
--- /dev/null
+++ b/src/test/org/apache/sqoop/lib/TestClobRef.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.lib;
+
+import java.io.*;
+
+import org.apache.sqoop.testutil.BaseSqoopTestCase;
+import org.apache.sqoop.testutil.CommonArgs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.io.LobFile;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test parsing of ClobRef objects.
+ */
+public class TestClobRef {
+
+ @Test
+ public void testEmptyStr() {
+ ClobRef r = ClobRef.parse("");
+ assertFalse(r.isExternal());
+ assertEquals("", r.toString());
+ }
+
+ @Test
+ public void testInline() throws IOException {
+ ClobRef r = ClobRef.parse("foo");
+ assertFalse(r.isExternal());
+ assertEquals("foo", r.toString());
+
+ Reader reader = r.getDataStream(null, null);
+ assertNotNull(reader);
+ char [] buf = new char[4096];
+ int chars = reader.read(buf, 0, 4096);
+ reader.close();
+
+ String str = new String(buf, 0, chars);
+ assertEquals("foo", str);
+ }
+
+ @Test
+ public void testEmptyFile() {
+ ClobRef r = ClobRef.parse("externalLob()");
+ assertFalse(r.isExternal());
+ assertEquals("externalLob()", r.toString());
+
+ r = ClobRef.parse("externalLob(lf,,0,0)");
+ assertTrue(r.isExternal());
+ assertEquals("externalLob(lf,,0,0)", r.toString());
+ }
+
+ @Test
+ public void testInlineNearMatch() {
+ ClobRef r = ClobRef.parse("externalLob(foo)bar");
+ assertFalse(r.isExternal());
+ assertEquals("externalLob(foo)bar", r.toString());
+
+ r = ClobRef.parse("externalLob(foo)");
+ assertFalse(r.isExternal());
+ assertEquals("externalLob(foo)", r.getData());
+
+ r = ClobRef.parse("externalLob(lf,foo)");
+ assertFalse(r.isExternal());
+ assertEquals("externalLob(lf,foo)", r.getData());
+
+ r = ClobRef.parse("externalLob(lf,foo,1,2)x");
+ assertFalse(r.isExternal());
+ assertEquals("externalLob(lf,foo,1,2)x", r.getData());
+ }
+
+ @Test
+ public void testExternal() throws IOException {
+ final String DATA = "This is the clob data!";
+ final String FILENAME = "clobdata";
+
+ doExternalTest(DATA, FILENAME);
+ }
+
+ @Test
+ public void testExternalSubdir() throws IOException {
+ final String DATA = "This is the clob data!";
+ final String FILENAME = "_lob/clobdata";
+
+ try {
+ doExternalTest(DATA, FILENAME);
+ } finally {
+ // remove dir we made.
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ Path lobDir = new Path(new Path(tmpDir), "_lob");
+ fs.delete(lobDir, true);
+ }
+ }
+
+ private void doExternalTest(final String data, final String filename)
+ throws IOException {
+
+ Configuration conf = new Configuration();
+ if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
+ conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
+ }
+ FileSystem fs = FileSystem.get(conf);
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+
+ Path tmpPath = new Path(tmpDir);
+ Path clobFile = new Path(tmpPath, filename);
+
+ // make any necessary parent dirs.
+ Path clobParent = clobFile.getParent();
+ if (!fs.exists(clobParent)) {
+ fs.mkdirs(clobParent);
+ }
+
+ LobFile.Writer lw = LobFile.create(clobFile, conf, true);
+ try {
+ long off = lw.tell();
+ long len = data.length();
+ Writer w = lw.writeClobRecord(len);
+ w.append(data);
+ w.close();
+ lw.close();
+
+ String refString = "externalLob(lf," + filename
+ + "," + off + "," + len + ")";
+ ClobRef clob = ClobRef.parse(refString);
+ assertTrue(clob.isExternal());
+ assertEquals(refString, clob.toString());
+ Reader r = clob.getDataStream(conf, tmpPath);
+ assertNotNull(r);
+
+ char [] buf = new char[4096];
+ int chars = r.read(buf, 0, 4096);
+ r.close();
+
+ String str = new String(buf, 0, chars);
+ assertEquals(data, str);
+ } finally {
+ fs.delete(clobFile, false);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestFieldFormatter.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/lib/TestFieldFormatter.java b/src/test/org/apache/sqoop/lib/TestFieldFormatter.java
new file mode 100644
index 0000000..9ac55e7
--- /dev/null
+++ b/src/test/org/apache/sqoop/lib/TestFieldFormatter.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.lib;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+
+/**
+ * Test that the field formatter works in a variety of configurations.
+ */
+public class TestFieldFormatter {
+
+ @Test
+ public void testAllEmpty() {
+ String result = FieldFormatter.escapeAndEnclose("",
+ new DelimiterSet(DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR,
+ DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR, false));
+ assertEquals("", result);
+ }
+
+ @Test
+ public void testNullArgs() {
+ assertNull(FieldFormatter.escapeAndEnclose(null,
+ new DelimiterSet('\"', DelimiterSet.NULL_CHAR, '\"', '\\', false)));
+ }
+
+ @Test
+ public void testBasicStr() {
+ String result = FieldFormatter.escapeAndEnclose("foo",
+ DelimiterSet.DEFAULT_DELIMITERS);
+ assertEquals("foo", result);
+ }
+
+ @Test
+ public void testEscapeSlash() {
+ String result = FieldFormatter.escapeAndEnclose("foo\\bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("foo\\\\bar", result);
+ }
+
+ @Test
+ public void testMustEnclose() {
+ String result = FieldFormatter.escapeAndEnclose("foo",
+ new DelimiterSet(',', '\n', '\"', DelimiterSet.NULL_CHAR, true));
+ assertEquals("\"foo\"", result);
+ }
+
+ @Test
+ public void testEncloseComma1() {
+ String result = FieldFormatter.escapeAndEnclose("foo,bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("\"foo,bar\"", result);
+ }
+
+ @Test
+ public void testEncloseComma2() {
+ String result = FieldFormatter.escapeAndEnclose("foo,bar",
+ new DelimiterSet(',', ',', '\"', '\\', false));
+ assertEquals("\"foo,bar\"", result);
+ }
+
+ @Test
+ public void testNoNeedToEnclose() {
+ String result = FieldFormatter.escapeAndEnclose(
+ "just another string",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("just another string", result);
+ }
+
+ @Test
+ public void testCannotEnclose() {
+ // Can't enclose because encloser is nul.
+ // This should escape the comma instead.
+ String result = FieldFormatter.escapeAndEnclose("foo,bar",
+ new DelimiterSet(',', '\n', DelimiterSet.NULL_CHAR, '\\', false));
+
+ assertEquals("foo\\,bar", result);
+ }
+
+ @Test
+ public void testEmptyCharToEscapeString() {
+ // test what happens when the escape char is null. It should encode the
+ // null char.
+
+ char nul = DelimiterSet.NULL_CHAR;
+ String s = "" + nul;
+ assertEquals("\000", s);
+ }
+
+ @Test
+ public void testEscapeCentralQuote() {
+ String result = FieldFormatter.escapeAndEnclose("foo\"bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("foo\\\"bar", result);
+ }
+
+ @Test
+ public void testEscapeMultiCentralQuote() {
+ String result = FieldFormatter.escapeAndEnclose("foo\"\"bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("foo\\\"\\\"bar", result);
+ }
+
+ @Test
+ public void testDoubleEscape() {
+ String result = FieldFormatter.escapeAndEnclose("foo\\\"bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("foo\\\\\\\"bar", result);
+ }
+
+ @Test
+ public void testReverseEscape() {
+ String result = FieldFormatter.escapeAndEnclose("foo\"\\bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("foo\\\"\\\\bar", result);
+ }
+
+ @Test
+ public void testQuotedEncloser() {
+ String result = FieldFormatter.escapeAndEnclose("foo\",bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("\"foo\\\",bar\"", result);
+ }
+
+ @Test
+ public void testQuotedEscape() {
+ String result = FieldFormatter.escapeAndEnclose("foo\\,bar",
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ assertEquals("\"foo\\\\,bar\"", result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java b/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java
new file mode 100644
index 0000000..1e07d71
--- /dev/null
+++ b/src/test/org/apache/sqoop/lib/TestLargeObjectLoader.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.lib;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.sqoop.testutil.BaseSqoopTestCase;
+import org.apache.sqoop.testutil.CommonArgs;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.sqoop.testutil.MockResultSet;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Test deserialization of ClobRef and BlobRef fields.
+ */
+public class TestLargeObjectLoader {
+
+ protected Configuration conf;
+ protected LargeObjectLoader loader;
+ protected Path outDir;
+
+ @Before
+ public void setUp() throws IOException, InterruptedException {
+ conf = new Configuration();
+ if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
+ conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
+ }
+ String tmpDir = System.getProperty("test.build.data", "/tmp/");
+ this.outDir = new Path(System.getProperty("java.io.tmpdir"));
+ FileSystem fs = FileSystem.get(conf);
+ if (fs.exists(outDir)) {
+ fs.delete(outDir, true);
+ }
+ fs.mkdirs(outDir);
+
+ loader = new LargeObjectLoader(conf, outDir);
+ }
+
+ @Test
+ public void testReadClobRef()
+ throws IOException, InterruptedException, SQLException {
+ // This should give us an inline CLOB.
+ ResultSet resultSet = new MockResultSet();
+ ClobRef clob = loader.readClobRef(0, resultSet);
+ assertNotNull(clob);
+ assertFalse(clob.isExternal());
+ assertEquals(MockResultSet.CLOB_DATA, clob.toString());
+
+ // LOBs bigger than 4 bytes are now external.
+ conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4);
+ clob = loader.readClobRef(0, resultSet);
+ assertNotNull(clob);
+ assertTrue(clob.isExternal());
+ loader.close();
+ Reader r = clob.getDataStream(conf, outDir);
+ char [] buf = new char[4096];
+ int chars = r.read(buf, 0, 4096);
+ r.close();
+ String str = new String(buf, 0, chars);
+ assertEquals(MockResultSet.CLOB_DATA, str);
+ }
+
+ @Test
+ public void testReadBlobRef()
+ throws IOException, InterruptedException, SQLException {
+ // This should give us an inline BLOB.
+ ResultSet resultSet = new MockResultSet();
+ BlobRef blob = loader.readBlobRef(0, resultSet);
+ assertNotNull(blob);
+ assertFalse(blob.isExternal());
+ byte [] data = blob.getData();
+ byte [] blobData = MockResultSet.blobData();
+ assertEquals(blobData.length, data.length);
+ for (int i = 0; i < data.length; i++) {
+ assertEquals(blobData[i], data[i]);
+ }
+
+ // LOBs bigger than 4 bytes are now external.
+ conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4);
+ blob = loader.readBlobRef(0, resultSet);
+ assertNotNull(blob);
+ assertTrue(blob.isExternal());
+ loader.close();
+ InputStream is = blob.getDataStream(conf, outDir);
+ byte [] buf = new byte[4096];
+ int bytes = is.read(buf, 0, 4096);
+ is.close();
+
+ assertEquals(blobData.length, bytes);
+ for (int i = 0; i < bytes; i++) {
+ assertEquals(blobData[i], buf[i]);
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/lib/TestRecordParser.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/lib/TestRecordParser.java b/src/test/org/apache/sqoop/lib/TestRecordParser.java
new file mode 100644
index 0000000..d6844c1
--- /dev/null
+++ b/src/test/org/apache/sqoop/lib/TestRecordParser.java
@@ -0,0 +1,450 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.sqoop.lib;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import static org.junit.Assert.fail;
+
+
+/**
+ * Test that the record parser works in a variety of configurations.
+ */
+public class TestRecordParser {
+
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
+
+ private void assertListsEqual(String msg, List<String> expected,
+ List<String> actual) {
+ if (expected == null && actual != null) {
+ if (null == msg) {
+ msg = "expected null list";
+ }
+
+ fail(msg);
+ } else if (expected != null && actual == null) {
+ if (null == msg) {
+ msg = "expected non-null list";
+ }
+
+ fail(msg);
+ } else if (expected == null && actual == null) {
+ return; // ok. Both null; nothing to do.
+ }
+
+ assert(null != expected);
+ assert(null != actual);
+
+ int expectedLen = expected.size();
+ int actualLen = actual.size();
+
+ if (expectedLen != actualLen) {
+ if (null == msg) {
+ msg = "Expected list of length " + expectedLen
+ + "; got " + actualLen;
+ }
+
+ fail(msg);
+ }
+
+ // Check the list contents.
+ for (int i = 0; i < expectedLen; i++) {
+ String expectedElem = expected.get(i);
+ String actualElem = actual.get(i);
+
+ if (expectedElem == null) {
+ if (actualElem != null) {
+ if (null == msg) {
+ msg = "Expected null element at position " + i
+ + "; got [" + actualElem + "]";
+ }
+
+ fail(msg);
+ }
+ } else if (!expectedElem.equals(actualElem)) {
+ if (null == msg) {
+ msg = "Expected [" + expectedElem + "] at position " + i
+ + "; got [" + actualElem + "]";
+ }
+
+ fail(msg);
+ }
+ }
+ }
+
+ private List<String> list(String [] items) {
+
+ if (null == items) {
+ return null;
+ }
+
+ ArrayList<String> asList = new ArrayList<String>();
+ for (int i = 0; i < items.length; i++) {
+ asList.add(items[i]);
+ }
+
+ return asList;
+ }
+
+ @Test
+ public void testEmptyLine() throws RecordParser.ParseError {
+ // an empty line should return no fields.
+
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { };
+ assertListsEqual(null, list(strings), parser.parseRecord(""));
+ }
+
+ @Test
+ public void testJustEOR() throws RecordParser.ParseError {
+ // a line with just a newline char should return a single zero-length field.
+
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "" };
+ assertListsEqual(null, list(strings), parser.parseRecord("\n"));
+ }
+
+ @Test
+ public void testOneField() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the field" };
+ assertListsEqual(null, list(strings), parser.parseRecord("the field"));
+ }
+
+ @Test
+ public void testOneField2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the field" };
+ assertListsEqual(null, list(strings), parser.parseRecord("the field\n"));
+ }
+
+ @Test
+ public void testQuotedField1() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the field" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the field\"\n"));
+ }
+
+ @Test
+ public void testQuotedField2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the field" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the field\""));
+ }
+
+ @Test
+ public void testQuotedField3() throws RecordParser.ParseError {
+ // quoted containing EOF
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the ,field" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the ,field\""));
+ }
+
+ @Test
+ public void testQuotedField4() throws RecordParser.ParseError {
+ // quoted containing multiple EOFs
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the ,,field" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the ,,field\""));
+ }
+
+ @Test
+ public void testQuotedField5() throws RecordParser.ParseError {
+ // quoted containing EOF and EOR
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the ,\nfield" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the ,\nfield\""));
+ }
+
+ @Test
+ public void testQuotedField6() throws RecordParser.ParseError {
+ // quoted containing EOR
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the \nfield" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the \nfield\""));
+ }
+
+ @Test
+ public void testQuotedField7() throws RecordParser.ParseError {
+ // quoted containing multiple EORs
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the \n\nfield" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the \n\nfield\""));
+ }
+
+ @Test
+ public void testQuotedField8() throws RecordParser.ParseError {
+ // quoted containing escaped quoted char
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the \"field" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"the \\\"field\""));
+ }
+
+ @Test
+ public void testUnquotedEscape1() throws RecordParser.ParseError {
+ // field without quotes with an escaped EOF char.
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the ,field" };
+ assertListsEqual(null, list(strings), parser.parseRecord("the \\,field"));
+ }
+
+ @Test
+ public void testUnquotedEscape2() throws RecordParser.ParseError {
+ // field without quotes with an escaped escape char.
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "the \\field" };
+ assertListsEqual(null, list(strings), parser.parseRecord("the \\\\field"));
+ }
+
+ @Test
+ public void testTwoFields1() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings), parser.parseRecord("field1,field2"));
+ }
+
+ @Test
+ public void testTwoFields2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("field1,field2\n"));
+ }
+
+ @Test
+ public void testTwoFields3() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"field1\",field2\n"));
+ }
+
+ @Test
+ public void testTwoFields4() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("field1,\"field2\"\n"));
+ }
+
+ @Test
+ public void testTwoFields5() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("field1,\"field2\""));
+ }
+
+ @Test
+ public void testRequiredQuotes0() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', true));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"field1\",\"field2\"\n"));
+ }
+
+ @Test
+ public void testRequiredQuotes1() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', true));
+ String [] strings = { "field1", "field2" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("\"field1\",\"field2\""));
+ }
+
+ @Test
+ public void testRequiredQuotes2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', true));
+
+ thrown.expect(RecordParser.ParseError.class);
+ thrown.reportMissingExceptionWithMessage("Expected parse error for required quotes");
+ parser.parseRecord("\"field1\",field2");
+ }
+
+ @Test
+ public void testRequiredQuotes3() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', true));
+
+ thrown.expect(RecordParser.ParseError.class);
+ thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes");
+ parser.parseRecord("field1,\"field2\"");
+ }
+
+ @Test
+ public void testRequiredQuotes4() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', true));
+
+ thrown.expect(RecordParser.ParseError.class);
+ thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes");
+ parser.parseRecord("field1,\"field2\"\n");
+ }
+
+ @Test
+ public void testNull() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', true));
+ String input = null;
+
+ thrown.expect(RecordParser.ParseError.class);
+ thrown.reportMissingExceptionWithMessage("Expected ParseError for null string");
+ parser.parseRecord(input);
+ }
+
+
+ @Test
+ public void testEmptyFields1() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "", ""};
+ assertListsEqual(null, list(strings), parser.parseRecord(","));
+ }
+
+ @Test
+ public void testEmptyFields2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "", "" };
+ assertListsEqual(null, list(strings), parser.parseRecord(",\n"));
+ }
+
+ @Test
+ public void testEmptyFields3() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "", "", "" };
+ assertListsEqual(null, list(strings), parser.parseRecord(",,\n"));
+ }
+
+ @Test
+ public void testEmptyFields4() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "", "foo", "" };
+ assertListsEqual(null, list(strings), parser.parseRecord(",foo,\n"));
+ }
+
+ @Test
+ public void testEmptyFields5() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "", "foo", "" };
+ assertListsEqual(null, list(strings), parser.parseRecord(",foo,"));
+ }
+
+ @Test
+ public void testEmptyFields6() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "foo", "" };
+ assertListsEqual(null, list(strings), parser.parseRecord("foo,"));
+ }
+
+ @Test
+ public void testTrailingText() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "foo", "bar" };
+ assertListsEqual(null, list(strings), parser.parseRecord("foo,bar\nbaz"));
+ }
+
+ @Test
+ public void testTrailingText2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "" };
+ assertListsEqual(null, list(strings), parser.parseRecord("\nbaz"));
+ }
+
+ @Test
+ public void testLeadingEscape() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', '\n', '\"', '\\', false));
+ String [] strings = { "\nbaz" };
+ assertListsEqual(null, list(strings), parser.parseRecord("\\\nbaz"));
+ }
+
+ @Test
+ public void testEofIsEor() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', ',', '\"', '\\', false));
+ String [] strings = { "three", "different", "fields" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("three,different,fields"));
+ }
+
+ @Test
+ public void testEofIsEor2() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', ',', '\"', '\\', false));
+ String [] strings = { "three", "different", "fields" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("three,\"different\",fields"));
+ }
+
+ @Test
+ public void testRepeatedParse() throws RecordParser.ParseError {
+ RecordParser parser = new RecordParser(
+ new DelimiterSet(',', ',', '\"', '\\', false));
+ String [] strings = { "three", "different", "fields" };
+ assertListsEqual(null, list(strings),
+ parser.parseRecord("three,\"different\",fields"));
+
+ String [] strings2 = { "foo", "bar" };
+ assertListsEqual(null, list(strings2),
+ parser.parseRecord("foo,\"bar\""));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java b/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java
index fd72ef4..8e16324 100644
--- a/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java
+++ b/src/test/org/apache/sqoop/manager/TestDefaultManagerFactory.java
@@ -18,8 +18,8 @@
package org.apache.sqoop.manager;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.metastore.JobData;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.metastore.JobData;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.hadoop.conf.Configuration;
import org.junit.Test;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/org/apache/sqoop/manager/TestMainframeManager.java
----------------------------------------------------------------------
diff --git a/src/test/org/apache/sqoop/manager/TestMainframeManager.java b/src/test/org/apache/sqoop/manager/TestMainframeManager.java
index 9359ac4..97e48e8 100644
--- a/src/test/org/apache/sqoop/manager/TestMainframeManager.java
+++ b/src/test/org/apache/sqoop/manager/TestMainframeManager.java
@@ -36,13 +36,11 @@ import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import com.cloudera.sqoop.ConnFactory;
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.manager.ConnManager;
-import com.cloudera.sqoop.manager.ImportJobContext;
-import com.cloudera.sqoop.metastore.JobData;
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.util.ImportException;
+import org.apache.sqoop.ConnFactory;
+import org.apache.sqoop.SqoopOptions;
+import org.apache.sqoop.metastore.JobData;
+import org.apache.sqoop.testutil.BaseSqoopTestCase;
+import org.apache.sqoop.util.ImportException;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;