You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by an...@apache.org on 2018/01/18 14:04:06 UTC
[20/32] sqoop git commit: SQOOP-3273: Removing com.cloudera.sqoop
packages
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/io/TestLobFile.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/io/TestLobFile.java b/src/test/com/cloudera/sqoop/io/TestLobFile.java
deleted file mode 100644
index ac461fd..0000000
--- a/src/test/com/cloudera/sqoop/io/TestLobFile.java
+++ /dev/null
@@ -1,642 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.io;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.Reader;
-import java.io.Writer;
-import java.net.URI;
-import java.nio.CharBuffer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.crypto.CryptoOutputStream;
-import org.apache.hadoop.crypto.JceAesCtrCryptoCodec;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Before;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Answers.CALLS_REAL_METHODS;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.withSettings;
-
-import org.junit.Rule;
-import org.junit.rules.ExpectedException;
-
-/**
- * Test the LobFile reader/writer implementation.
- */
-public class TestLobFile {
-
- public static final Log LOG = LogFactory.getLog(
- TestLobFile.class.getName());
-
- public static final Path TEMP_BASE_DIR;
-
- static {
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
- if (!tmpDir.endsWith(File.separator)) {
- tmpDir = tmpDir + File.separator;
- }
-
- TEMP_BASE_DIR = new Path(new Path(tmpDir), "lobtest");
- }
-
- private Configuration conf;
- private FileSystem fs;
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- conf.set("fs.default.name", "file:///");
-
- fs = FileSystem.getLocal(conf);
- fs.mkdirs(TEMP_BASE_DIR);
- }
-
- private long[] writeClobFile(Path p, String codec,
- String... records) throws Exception {
- if (fs.exists(p)) {
- fs.delete(p, false);
- }
-
- // memorize the offsets of each record we write.
- long[] offsets = new long[records.length];
-
- // Create files with four entries per index segment.
- LobFile.Writer writer = LobFile.create(p, conf, true, codec, 4);
-
- int i = 0;
- for (String r : records) {
- offsets[i++] = writer.tell();
- Writer w = writer.writeClobRecord(r.length());
- w.write(r);
- w.close();
- }
-
- writer.close();
- return offsets;
- }
-
- private void verifyClobFile(Path p, String... expectedRecords)
- throws Exception {
-
- LobFile.Reader reader = LobFile.open(p, conf);
-
- int recNum = 0;
-
- while (reader.next()) {
- // We should have a record of the same length as the expected one.
- String expected = expectedRecords[recNum];
- assertTrue(reader.isRecordAvailable());
- assertEquals(expected.length(), reader.getRecordLen());
- Reader r = reader.readClobRecord();
-
- // Read in the record and assert that we got enough characters out.
- CharBuffer buf = CharBuffer.allocate(expected.length());
- int bytesRead = 0;
- while (bytesRead < expected.length()) {
- int thisRead = r.read(buf);
- LOG.info("performed read of " + thisRead + " chars");
- if (-1 == thisRead) {
- break;
- }
-
- bytesRead += thisRead;
- }
-
- LOG.info("Got record of " + bytesRead + " chars");
- assertEquals(expected.length(), bytesRead);
- char[] charData = buf.array();
- String finalRecord = new String(charData);
- assertEquals(expected, finalRecord);
-
- recNum++;
- }
-
- // Check that we got everything.
- assertEquals(expectedRecords.length, recNum);
-
- reader.close();
-
- thrown.expect(IOException.class);
- thrown.reportMissingExceptionWithMessage("Expected IOException calling next after close");
- reader.next();
-
- // A second close shouldn't hurt anything. This should be a no-op.
- reader.close();
- }
-
- private void runClobFileTest(Path p, String codec,
- String... records) throws Exception {
- writeClobFile(p, codec, records);
- verifyClobFile(p, records);
- fs.delete(p, false);
- }
-
- @Test
- public void testEmptyRecord() throws Exception {
- runClobFileTest(new Path(TEMP_BASE_DIR, "empty.lob"), null);
- }
-
- @Test
- public void testSingleRecord() throws Exception {
- runClobFileTest(new Path(TEMP_BASE_DIR, "single.lob"),
- null, "this is a single record!");
- }
-
- @Test
- public void testMultiRecords() throws Exception {
- runClobFileTest(new Path(TEMP_BASE_DIR, "multi.lob"),
- CodecMap.NONE,
- "this is the first record",
- "this is the second record. I assure you that this record is long.",
- "yet one more record graces this file.");
- }
-
- @Test
- public void testMultiIndexSegments() throws Exception {
- // Test that we can use multiple IndexSegments.
- runClobFileTest(new Path(TEMP_BASE_DIR, "multi-index.lob"),
- null,
- "this is the first record",
- "this is the second record. I assure you that this record is long.",
- "record number three",
- "last one in first index segment",
- "first in the second index segment",
- "yet one more record graces this file.");
- }
-
- /**
- * Run a test where we read only a fraction of the first record,
- * but then read the second record completely. Verify that we
- * can re-align on a record boundary correctly. This test requires
- * at least 3 records.
- *
- * @param p the path to the file to create.
- * @param firstLine the first line of the first reord
- * @param records All of the records to write to the file.
- */
- private void runLineAndRecordTest(Path p, String firstLine,
- String... records) throws Exception {
-
- assertTrue("This test requires 3+ records", records.length > 2);
-
- writeClobFile(p, null, records);
-
- LobFile.Reader reader = LobFile.open(p, conf);
-
- // We should not yet be aligned.
- assertFalse(reader.isRecordAvailable());
- assertTrue(reader.next());
- // Now we should be.
- assertTrue(reader.isRecordAvailable());
-
- // Read just one line from the record.
- Reader r = reader.readClobRecord();
- BufferedReader br = new BufferedReader(r);
- String line = br.readLine();
- assertEquals(firstLine, line);
-
- br.close();
- r.close();
-
- // We should no longer be aligned on a record start.
- assertFalse(reader.isRecordAvailable());
-
- // We should now be able to get to record two.
- assertTrue(reader.next());
-
- // This should be nicely aligned even if the first record was not
- // completely consumed by a client.
- r = reader.readClobRecord();
- CharBuffer buf = CharBuffer.allocate(records[1].length());
- r.read(buf);
- r.close();
- char[] chars = buf.array();
- String s = new String(chars);
- assertEquals(records[1], s);
-
- // Close the reader before we consume the entire file.
- reader.close();
- assertFalse(reader.isRecordAvailable());
- }
-
- @Test
- public void testVeryShortRead() throws Exception {
- // Read only a small fraction of a record, ensure that we can
- // read the next record, even when we've left more than a 16-byte
- // quantity in the readahead buffer.
-
- Path p = new Path(TEMP_BASE_DIR, "shortread.lob");
- final String FIRST_LINE = "line1";
- final String SECOND_LINE =
- "This contains much more in the record than just one line.";
- final String RECORD2 = "here is the second record.";
- final String RECORD3 = "The 3rd record, which we won't actually read.";
-
- runLineAndRecordTest(p, FIRST_LINE,
- FIRST_LINE + "\n" + SECOND_LINE,
- RECORD2,
- RECORD3);
-
- }
-
- @Test
- public void testIncompleteOverread() throws Exception {
- // Read most of the first record so that we partially consume the
- // next record start mark; make sure we realign properly.
-
- Path p = new Path(TEMP_BASE_DIR, "longread.lob");
- final String FIRST_LINE = "this is a really long line of text to read!";
- final String SECOND_LINE = "not this.";
- final String RECORD2 = "Here is yet another record to verify.";
- final String RECORD3 = "Nobody cares about record 3.";
-
- runLineAndRecordTest(p, FIRST_LINE,
- FIRST_LINE + "\n" + SECOND_LINE,
- RECORD2,
- RECORD3);
- }
-
- @Test
- public void testSeekToRecord() throws Exception {
- // Seek past the first two records and read the third.
-
- Path p = new Path(TEMP_BASE_DIR, "seek.lob");
- String[] records = {
- "this is the first record!",
- "here comes record number two. It is a bit longer.",
- "this is the third record. we can read it.",
- };
-
- // Write the file and memorize when the third record starts.
- LobFile.Writer writer = LobFile.create(p, conf, true);
-
- int recNum = 0;
- long rec3Start = 0;
- for (String r : records) {
- Writer w = writer.writeClobRecord(r.length());
- w.write(r);
- w.close();
- writer.finishRecord();
- if (recNum == 1) {
- rec3Start = writer.tell();
- LOG.info("Record three start: " + rec3Start);
- }
- recNum++;
- }
-
- writer.close();
-
- // Now reopen the file for read, seek to the third record, and get it.
- LobFile.Reader reader = LobFile.open(p, conf);
- reader.seek(rec3Start);
- assertTrue(reader.next());
- assertTrue(reader.isRecordAvailable());
- assertEquals(rec3Start, reader.getRecordOffset());
-
- Reader r = reader.readClobRecord();
- CharBuffer buf = CharBuffer.allocate(records[2].length());
- r.read(buf);
- r.close();
- char[] chars = buf.array();
- String s = new String(chars);
- assertEquals(records[2], s);
-
- r.close();
- reader.close();
- }
-
-
- /**
- * Verifies that the next record in the LobFile is the expected one.
- */
- private void verifyNextRecord(LobFile.Reader reader, long expectedId,
- String expectedRecord) throws Exception {
- assertTrue(reader.next());
- assertTrue(reader.isRecordAvailable());
- assertEquals(expectedId, reader.getRecordId());
-
- Reader r = reader.readClobRecord();
- CharBuffer buf = CharBuffer.allocate(expectedRecord.length());
- int bytesRead = 0;
- while (bytesRead < expectedRecord.length()) {
- int thisRead = r.read(buf);
- if (-1 == thisRead) {
- break;
- }
-
- bytesRead += thisRead;
- }
-
- LOG.info("Got record of " + bytesRead + " chars");
- assertEquals(expectedRecord.length(), bytesRead);
-
- char[] charData = buf.array();
- String finalRecord = new String(charData);
- assertEquals(expectedRecord, finalRecord);
- }
-
- @Test
- public void testManySeeks() throws Exception {
- // Test that we can do gymnastics with seeking between records.
-
- Path p = new Path(TEMP_BASE_DIR, "manyseeks.lob");
-
- String[] records = {
- "first record",
- "second record",
- "the third record",
- "rec4 is the last in IndexSeg 0",
- "rec5 is first in IndexSeg 1",
- "rec6 is yet another record",
- "rec7 is starting to feel boring",
- "rec8 is at the end of seg 1",
- "rec9 is all by itself in seg 2",
- };
-
- // Write the records to a file, save their offsets.
- long[] offsets = writeClobFile(p, null, records);
-
- // Sanity check that we can stream the file.
- verifyClobFile(p, records);
-
- // Open a handle to the file.
- LobFile.Reader reader = LobFile.open(p, conf);
-
- // Seeking to offset 0 should return the first record.
- reader.seek(0);
- verifyNextRecord(reader, 0, records[0]);
-
- // Seek to the last item in the first IndexSegment.
- reader.seek(offsets[3]);
- verifyNextRecord(reader, 3, records[3]);
-
- // Seek to just ahead of that same record.
- reader.seek(offsets[3] - 10);
- verifyNextRecord(reader, 3, records[3]);
-
- // Seek (backwards) to the first record.
- reader.seek(offsets[0]);
- verifyNextRecord(reader, 0, records[0]);
-
- // Seek to first record in second IndexSegment.
- reader.seek(offsets[4]);
- verifyNextRecord(reader, 4, records[4]);
-
- // Move backwards.
- reader.seek(0);
-
- // Seek to "no man's land" between last offset in first IndexSeg
- // and the first offset in second IndexSegment. Result should be
- // the first record in second InexSegment.
- reader.seek(offsets[4] - 10);
- verifyNextRecord(reader, 4, records[4]);
-
- // Seek to past the last record. No record should be returned.
- reader.seek(offsets[8] + 4);
- assertFalse("Found a record past last record start.", reader.next());
-
- // Seek to somewhere in the middle of IndexSegment 0.
- // This should recover just fine.
- reader.seek(offsets[2]);
- verifyNextRecord(reader, 2, records[2]);
-
- // Seek to last record in IndexSegment 1.
- reader.seek(offsets[3] - 1);
- verifyNextRecord(reader, 3, records[3]);
-
- // And make sure that iteration picks up naturally from there.
- verifyNextRecord(reader, 4, records[4]);
-
- // Seek well past the end of the file. No record should be returned.
- reader.seek(50000000);
- assertFalse("Found a record past expected end-of-file", reader.next());
-
- // Seek to somewhere in the index.
- reader.seek(offsets[8] + 32);
- assertFalse("Found a record past beginning of index", reader.next());
-
- // Seek to the last record (exact hit). This is a singleton IndexSegment.
- reader.seek(offsets[8]);
- verifyNextRecord(reader, 8, records[8]);
-
- // Seek to no-man's-land ahead of last record.
- reader.seek(offsets[8] - 3);
- verifyNextRecord(reader, 8, records[8]);
-
- reader.close();
- }
-
- /**
- * Verifies that a record to be read from a lob file has
- * as many bytes as we expect, and that the bytes are what we
- * expect them to be. Assumes that the bytes are such that
- * input[i] == i + offset.
- *
- * @param reader the LobFile reader to consume data from
- * @param expectedDeclaredLen the size we expect the LobFile to declare
- * its record length as.
- * @param expectedActualLen the true number of bytes we expect to read in
- * the record.
- * @param offset the offset amount for each of the elements of the array.
- */
- private void verifyBlobRecord(LobFile.Reader reader,
- long expectedDeclaredLen, long expectedActualLen,
- int offset) throws Exception {
-
- assertTrue(reader.next());
- assertTrue(reader.isRecordAvailable());
- assertEquals(expectedDeclaredLen, reader.getRecordLen());
-
- InputStream is = reader.readBlobRecord();
-
- byte[] bytes = new byte[(int) expectedActualLen];
- int numRead = is.read(bytes);
- assertEquals(expectedActualLen, numRead);
-
- for (int i = 0; i < numRead; i++) {
- assertEquals(i + offset, (int) bytes[i]);
- }
-
- is.close();
- }
-
- /**
- * Write a binary record to a LobFile. This allows the declared length
- * of the record to disagree with the actual length (the actual length
- * should be >= the declared length).
- * The record written will have values v[i] = i + offset.
- *
- * @param writer the LobFile writer to put the record into
- * @param declaredLen the length value written into the file itself
- * @param actualLen the true number of bytes to write
- * @param offset an amount to adjust each record's byte values by.
- */
- private void writeBlobRecord(LobFile.Writer writer, long declaredLen,
- long actualLen, int offset) throws Exception {
- OutputStream os = writer.writeBlobRecord(declaredLen);
- for (int i = 0; i < actualLen; i++) {
- os.write(i + offset);
- }
-
- os.close();
- writer.finishRecord();
- }
-
- /**
- * Verifies a number of records that all have the same declared
- * and actual record lengths.
- *
- * @param p the path to the LobFile to open
- * @param numRecords the number of records to expect
- * @param declaredLen the declared length of each record in the file
- * @param actualLen the true number of bytes we expect to read per record.
- */
- private void verifyBlobRecords(Path p, int numRecords,
- long declaredLen, long actualLen) throws Exception {
-
- LobFile.Reader reader = LobFile.open(p, conf);
- for (int i = 0; i < numRecords; i++) {
- verifyBlobRecord(reader, declaredLen, actualLen, i);
- }
- assertFalse(reader.next());
- reader.close();
- }
-
- @Test
- public void testBinaryRecords() throws Exception {
- // Write a BLOB file and read it all back.
-
- final long RECORD_LEN = 32;
- final int NUM_RECORDS = 2;
- Path p = new Path(TEMP_BASE_DIR, "binary.lob");
- LobFile.Writer writer = LobFile.create(p, conf);
-
- for (int i = 0; i < NUM_RECORDS; i++) {
- writeBlobRecord(writer, RECORD_LEN, RECORD_LEN, i);
- }
-
- writer.close();
-
- // Now check the read-back on those records.
- verifyBlobRecords(p, NUM_RECORDS, RECORD_LEN, RECORD_LEN);
- }
-
- @Test
- public void testOverLengthBinaryRecord() throws Exception {
- // Write a record with a declared length shorter than the
- // actual length, and read it back.
-
- final long ACTUAL_RECORD_LEN = 48;
- final long DECLARED_RECORD_LEN = 32;
- final int NUM_RECORDS = 2;
-
- Path p = new Path(TEMP_BASE_DIR, "overlength.lob");
- LobFile.Writer writer = LobFile.create(p, conf);
-
- for (int i = 0; i < NUM_RECORDS; i++) {
- writeBlobRecord(writer, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN, i);
- }
-
- writer.close();
-
- // Now read them back.
- verifyBlobRecords(p, NUM_RECORDS, DECLARED_RECORD_LEN, ACTUAL_RECORD_LEN);
- }
-
- private void runCompressedTest(String codec) throws Exception {
- LOG.info("Testing with codec: " + codec);
- Path p = new Path(TEMP_BASE_DIR, "compressed-" + codec + ".lob");
- String[] records = {
- "this is the first record, It should be compressed a lot!",
- "record 2 record 2 record 2 record 2 2 2 2 2 2 2 2 2 2 2 2",
- "and a third and a third yes this is the third",
- };
-
- runClobFileTest(p, codec, records);
- }
-
- @Test
- public void testCompressedFile() throws Exception {
- // Test all the various compression codecs.
-
- // The following values for 'codec' should pass.
- runCompressedTest(null);
- runCompressedTest(CodecMap.NONE);
- runCompressedTest(CodecMap.DEFLATE);
-
- thrown.expect(UnsupportedCodecException.class);
- thrown.reportMissingExceptionWithMessage("Expected UnsupportedCodecException for lzo");
- runCompressedTest(CodecMap.LZO);
- }
-
- @Test
- public void testCryptoOutputStreamClosingDoesNotThrowExceptionAndClosedProperly() throws Exception {
- // Tests that closing CryptoOutputStream doesn't throw exception neither with Java 7 nor with Java 8
- // For a detailed explanation see SQOOP-3243
- CryptoOutputStream cryptoOutputStream = createCryptoOutputStream();
- FSDataOutputStream wrappedCryptoOutputStream = new FSDataOutputStream(cryptoOutputStream, null);
-
- Path mockPath = spy(new Path("file://" + TEMP_BASE_DIR, "binary.lob"));
- FileSystem mockFileSystem = mock(FileSystem.class, withSettings().defaultAnswer(CALLS_REAL_METHODS.get()));
-
- doReturn(mockFileSystem).when(mockPath).getFileSystem(conf);
- doReturn(null).when(mockFileSystem).getWorkingDirectory();
- doReturn(wrappedCryptoOutputStream).when(mockFileSystem).create(mockPath);
- doReturn(new URI("file:///")).when(mockFileSystem).getUri();
-
- LobFile.Writer writer = LobFile.create(mockPath, conf);
-
- writer.close();
-
- verify(cryptoOutputStream).close();
- }
-
-
- public CryptoOutputStream createCryptoOutputStream() throws Exception {
- final byte[] BYTES = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16};
- Path p = new Path(TEMP_BASE_DIR, "binary.lob");
-
- FSDataOutputStream fsDataOutputStream = fs.create(p);
- CryptoOutputStream cryptoOutputStream = spy(new CryptoOutputStream(fsDataOutputStream, new JceAesCtrCryptoCodec(), 512, BYTES, BYTES));
-
- return cryptoOutputStream;
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/io/TestNamedFifo.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/io/TestNamedFifo.java b/src/test/com/cloudera/sqoop/io/TestNamedFifo.java
deleted file mode 100644
index b11bbee..0000000
--- a/src/test/com/cloudera/sqoop/io/TestNamedFifo.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.io;
-
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.io.OutputStreamWriter;
-
-import org.apache.hadoop.util.StringUtils;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.Shell;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-/**
- * Test the named fifo utility.
- */
-public class TestNamedFifo {
-
- public static final Log LOG = LogFactory.getLog(
- TestNamedFifo.class.getName());
-
- public static final Path TEMP_BASE_DIR;
-
- static {
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
- if (!tmpDir.endsWith(File.separator)) {
- tmpDir = tmpDir + File.separator;
- }
-
- TEMP_BASE_DIR = new Path(new Path(tmpDir), "namedfifo");
- }
-
- private Configuration conf;
- private FileSystem fs;
-
- @Before
- public void setUp() throws Exception {
- conf = new Configuration();
- conf.set("fs.default.name", "file:///");
-
- fs = FileSystem.getLocal(conf);
- fs.mkdirs(TEMP_BASE_DIR);
- }
-
- static final String MSG = "THIS IS THE MESSAGE\n";
- static final String MSG2 = "Here is a follow-up.\n";
-
- private static class ReaderThread extends Thread {
- private File file;
- private IOException exception;
-
- public ReaderThread(File f) {
- this.file = f;
- }
-
- /** return any exception during the run method. */
- public IOException getException() {
- return this.exception;
- }
-
- public void run() {
- BufferedReader r = null;
- try {
- r = new BufferedReader(new InputStreamReader(
- new FileInputStream(file)));
-
- // Assert that after a flush, we get back what we wrote.
- String line = r.readLine();
- if (!MSG.trim().equals(line)) {
- throw new IOException("Expected " + MSG.trim() + " but got "
- + line);
- }
-
- // Assert that after closing the writer, we get back what
- // we wrote again.
- line = r.readLine();
- if (null == line) {
- throw new IOException("line2 was null");
- } else if (!MSG2.trim().equals(line)) {
- throw new IOException("Expected " + MSG2.trim() + " but got "
- + line);
- }
- } catch (IOException ioe) {
- this.exception = ioe;
- } finally {
- if (null != r) {
- try {
- r.close();
- } catch (IOException ioe) {
- LOG.warn("Error closing reader: " + ioe);
- }
- }
- }
- }
- }
-
- private static class WriterThread extends Thread {
- private File file;
- private IOException exception;
-
- public WriterThread(File f) {
- this.file = f;
- }
-
- /** return any exception during the run method. */
- public IOException getException() {
- return this.exception;
- }
-
- public void run() {
- BufferedWriter w = null;
- try {
- w = new BufferedWriter(new OutputStreamWriter(
- new FileOutputStream(file)));
-
- w.write(MSG);
- w.flush();
-
- w.write(MSG2);
- } catch (IOException ioe) {
- this.exception = ioe;
- } finally {
- if (null != w) {
- try {
- w.close();
- } catch (IOException ioe) {
- LOG.warn("Error closing writer: " + ioe);
- }
- }
- }
- }
- }
-
- @Test
- public void testNamedFifo() throws Exception {
-
- if (Shell.WINDOWS) {
- // NamedFifo uses Linux specific commands like mknod
- // and mkfifo, so skip the test on Windows OS
- LOG.warn("Named FIFO is not supported on Windows. Skipping test");
- return;
- }
-
- File root = new File(TEMP_BASE_DIR.toString());
- File fifo = new File(root, "foo-fifo");
-
- NamedFifo nf = new NamedFifo(fifo);
- nf.create();
-
- File returned = nf.getFile();
-
- // These should be the same object.
- assertEquals(fifo, returned);
-
- ReaderThread rt = new ReaderThread(returned);
- WriterThread wt = new WriterThread(returned);
-
- rt.start();
- wt.start();
-
- rt.join();
- wt.join();
-
- IOException rex = rt.getException();
- IOException wex = wt.getException();
-
- if (null != rex) {
- LOG.error("reader exception: " + StringUtils.stringifyException(rex));
- }
-
- if (null != wex) {
- LOG.error("writer exception: " + StringUtils.stringifyException(wex));
- }
-
- assertNull(rex);
- assertNull(wex);
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java b/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java
deleted file mode 100644
index 8b2b1e5..0000000
--- a/src/test/com/cloudera/sqoop/io/TestSplittableBufferedWriter.java
+++ /dev/null
@@ -1,318 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.io;
-
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.util.zip.GZIPInputStream;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.compress.GzipCodec;
-
-import com.cloudera.sqoop.testutil.ImportJobTestCase;
-
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-/**
- * Test that the splittable buffered writer system works.
- */
-public class TestSplittableBufferedWriter {
-
- public static final Log LOG = LogFactory.getLog(
- TestSplittableBufferedWriter.class.getName());
-
- private String getWriteDir() {
- return new File(ImportJobTestCase.TEMP_BASE_DIR,
- "bufferedWriterTest").toString();
- }
-
- private Path getWritePath() {
- return new Path(ImportJobTestCase.TEMP_BASE_DIR, "bufferedWriterTest");
- }
-
- /** Create the directory where we'll write our test files to; and
- * make sure it has no files in it.
- */
- private void ensureEmptyWriteDir() throws IOException {
- FileSystem fs = FileSystem.getLocal(getConf());
- Path writeDir = getWritePath();
-
- fs.mkdirs(writeDir);
-
- FileStatus [] stats = fs.listStatus(writeDir);
-
- for (FileStatus stat : stats) {
- if (stat.isDir()) {
- fail("setUp(): Write directory " + writeDir
- + " contains subdirectories");
- }
-
- LOG.debug("setUp(): Removing " + stat.getPath());
- if (!fs.delete(stat.getPath(), false)) {
- fail("setUp(): Could not delete residual file " + stat.getPath());
- }
- }
-
- if (!fs.exists(writeDir)) {
- fail("setUp: Could not create " + writeDir);
- }
- }
-
- @Before
- public void setUp() throws IOException {
- ensureEmptyWriteDir();
- }
-
- private Configuration getConf() {
- Configuration conf = new Configuration();
- conf.set("fs.default.name", "file:///");
- return conf;
- }
-
- /** Verifies contents of an InputStream. Closes the InputStream on
- * its way out. Fails the test if the file doesn't match the expected set
- * of lines.
- */
- private void verifyFileContents(InputStream is, String [] lines)
- throws IOException {
- BufferedReader r = new BufferedReader(new InputStreamReader(is));
- try {
- for (String expectedLine : lines) {
- String actualLine = r.readLine();
- assertNotNull(actualLine);
- assertEquals("Input line mismatch", expectedLine, actualLine);
- }
-
- assertNull("Stream had additional contents after expected line",
- r.readLine());
- } finally {
- r.close();
- try {
- is.close();
- } catch (IOException ioe) {
- // ignore IOE; may be closed by reader.
- }
- }
- }
-
- private void verifyFileExists(Path p) throws IOException {
- FileSystem fs = FileSystem.getLocal(getConf());
- assertTrue("File not found: " + p, fs.exists(p));
- }
-
- private void verifyFileDoesNotExist(Path p) throws IOException {
- FileSystem fs = FileSystem.getLocal(getConf());
- assertFalse("File found: " + p + " and we did not expect it", fs.exists(p));
- }
-
- @Test
- public void testNonSplittingTextFile() throws IOException {
- SplittingOutputStream os = new SplittingOutputStream(getConf(),
- getWritePath(), "nonsplit-", 0, null);
- try {
- SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
- try {
- w.allowSplit();
- w.write("This is a string!");
- w.newLine();
- w.write("This is another string!");
- w.allowSplit();
- } finally {
- w.close();
- }
- } finally {
- try {
- os.close();
- } catch (IOException ioe) {
- // Ignored; may be thrown because w is already closed.
- }
- }
-
- // Ensure we made exactly one file.
- Path writePath = new Path(getWritePath(), "nonsplit-00000");
- Path badPath = new Path(getWritePath(), "nonsplit-00001");
- verifyFileExists(writePath);
- verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file.
-
- // Now ensure all the data got there.
- String [] expectedLines = {
- "This is a string!",
- "This is another string!",
- };
-
- InputStream fis = new FileInputStream(new File(getWriteDir(),
- "nonsplit-00000"));
- try {
- verifyFileContents(fis, expectedLines);
- } finally {
- try {
- fis.close();
- } catch (IOException ioe) {
- // Ignored; may be closed by verifyFileContents().
- }
- }
- }
-
- @Test
- public void testNonSplittingGzipFile() throws IOException {
- SplittingOutputStream os = new SplittingOutputStream(getConf(),
- getWritePath(), "nonsplit-", 0, new GzipCodec());
- SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
- try {
- w.allowSplit();
- w.write("This is a string!");
- w.newLine();
- w.write("This is another string!");
- w.allowSplit();
- } finally {
- w.close();
- }
-
- // Ensure we made exactly one file.
- Path writePath = new Path(getWritePath(), "nonsplit-00000.gz");
- Path badPath = new Path(getWritePath(), "nonsplit-00001.gz");
- verifyFileExists(writePath);
- verifyFileDoesNotExist(badPath); // Ensure we didn't make a second file.
-
- // Now ensure all the data got there.
- String [] expectedLines = {
- "This is a string!",
- "This is another string!",
- };
- verifyFileContents(
- new GZIPInputStream(new FileInputStream(new File(getWriteDir(),
- "nonsplit-00000.gz"))), expectedLines);
- }
-
- @Test
- public void testSplittingTextFile() throws IOException {
- SplittingOutputStream os = new SplittingOutputStream(getConf(),
- getWritePath(), "split-", 10, null);
- try {
- SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
- try {
- w.allowSplit();
- w.write("This is a string!");
- w.newLine();
- w.write("This is another string!");
- } finally {
- w.close();
- }
- } finally {
- try {
- os.close();
- } catch (IOException ioe) {
- // Ignored; may be thrown because w is already closed.
- }
- }
-
- // Ensure we made exactly two files.
- Path writePath = new Path(getWritePath(), "split-00000");
- Path writePath2 = new Path(getWritePath(), "split-00001");
- Path badPath = new Path(getWritePath(), "split-00002");
- verifyFileExists(writePath);
- verifyFileExists(writePath2);
- verifyFileDoesNotExist(badPath); // Ensure we didn't make three files.
-
- // Now ensure all the data got there.
- String [] expectedLines0 = {
- "This is a string!",
- };
- InputStream fis = new FileInputStream(new File(getWriteDir(),
- "split-00000"));
- try {
- verifyFileContents(fis, expectedLines0);
- } finally {
- try {
- fis.close();
- } catch (IOException ioe) {
- // ignored; may be generated because fis closed in verifyFileContents.
- }
- }
-
- String [] expectedLines1 = {
- "This is another string!",
- };
- fis = new FileInputStream(new File(getWriteDir(), "split-00001"));
- try {
- verifyFileContents(fis, expectedLines1);
- } finally {
- try {
- fis.close();
- } catch (IOException ioe) {
- // Ignored; may be thrown because it's closed in verifyFileContents.
- }
- }
- }
-
- @Test
- public void testSplittingGzipFile() throws IOException {
- SplittingOutputStream os = new SplittingOutputStream(getConf(),
- getWritePath(), "splitz-", 3, new GzipCodec());
- SplittableBufferedWriter w = new SplittableBufferedWriter(os, true);
- try {
- w.write("This is a string!");
- w.newLine();
- w.write("This is another string!");
- } finally {
- w.close();
- }
-
- // Ensure we made exactly two files.
- Path writePath = new Path(getWritePath(), "splitz-00000.gz");
- Path writePath2 = new Path(getWritePath(), "splitz-00001.gz");
- Path badPath = new Path(getWritePath(), "splitz-00002.gz");
- verifyFileExists(writePath);
- verifyFileExists(writePath2);
- verifyFileDoesNotExist(badPath); // Ensure we didn't make three files.
-
- // Now ensure all the data got there.
- String [] expectedLines0 = {
- "This is a string!",
- };
- verifyFileContents(
- new GZIPInputStream(new FileInputStream(new File(getWriteDir(),
- "splitz-00000.gz"))), expectedLines0);
-
- String [] expectedLines1 = {
- "This is another string!",
- };
- verifyFileContents(
- new GZIPInputStream(new FileInputStream(new File(getWriteDir(),
- "splitz-00001.gz"))), expectedLines1);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestBlobRef.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/lib/TestBlobRef.java b/src/test/com/cloudera/sqoop/lib/TestBlobRef.java
deleted file mode 100644
index 0d010b0..0000000
--- a/src/test/com/cloudera/sqoop/lib/TestBlobRef.java
+++ /dev/null
@@ -1,155 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.lib;
-
-import java.io.*;
-
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.testutil.CommonArgs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import com.cloudera.sqoop.io.LobFile;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test that the BlobRef.parse() method does the right thing.
- * Note that we don't support inline parsing here; we only expect this to
- * really work for external BLOBs.
- */
-public class TestBlobRef {
-
- @Test
- public void testEmptyStr() {
- BlobRef r = BlobRef.parse("");
- assertFalse(r.isExternal());
- }
-
- @Test
- public void testInline() throws IOException {
- BlobRef r = BlobRef.parse("foo");
- assertFalse(r.isExternal());
- }
-
- @Test
- public void testEmptyFile() {
- BlobRef r = BlobRef.parse("externalLob()");
- assertFalse(r.isExternal());
-
- r = BlobRef.parse("externalLob(lf,,0,0)");
- assertTrue(r.isExternal());
- assertEquals("externalLob(lf,,0,0)", r.toString());
- }
-
- @Test
- public void testInlineNearMatch() {
- BlobRef r = BlobRef.parse("externalLob(foo)bar");
- assertFalse(r.isExternal());
-
- r = BlobRef.parse("externalLob(foo)");
- assertFalse(r.isExternal());
-
- r = BlobRef.parse("externalLob(lf,foo)");
- assertFalse(r.isExternal());
-
- r = BlobRef.parse("externalLob(lf,foo,1,2)x");
- assertFalse(r.isExternal());
- }
-
- @Test
- public void testExternal() throws IOException {
- final byte [] DATA = { 1, 2, 3, 4, 5 };
- final String FILENAME = "blobdata";
-
- doExternalTest(DATA, FILENAME);
- }
-
- @Test
- public void testExternalSubdir() throws IOException {
- final byte [] DATA = { 1, 2, 3, 4, 5 };
- final String FILENAME = "_lob/blobdata";
-
- try {
- doExternalTest(DATA, FILENAME);
- } finally {
- // remove dir we made.
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.getLocal(conf);
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
- Path lobDir = new Path(new Path(tmpDir), "_lob");
- fs.delete(lobDir, true);
- }
- }
-
- private void doExternalTest(final byte [] data, final String filename)
- throws IOException {
-
- Configuration conf = new Configuration();
- if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- FileSystem fs = FileSystem.get(conf);
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
-
- Path tmpPath = new Path(tmpDir);
- Path blobFile = new Path(tmpPath, filename);
-
- // make any necessary parent dirs.
- Path blobParent = blobFile.getParent();
- if (!fs.exists(blobParent)) {
- fs.mkdirs(blobParent);
- }
-
- LobFile.Writer lw = LobFile.create(blobFile, conf, false);
- try {
- long off = lw.tell();
- long len = data.length;
- OutputStream os = lw.writeBlobRecord(len);
- os.write(data, 0, data.length);
- os.close();
- lw.close();
-
- String refString = "externalLob(lf," + filename
- + "," + off + "," + len + ")";
- BlobRef blob = BlobRef.parse(refString);
- assertTrue(blob.isExternal());
- assertEquals(refString, blob.toString());
- InputStream is = blob.getDataStream(conf, tmpPath);
- assertNotNull(is);
-
- byte [] buf = new byte[4096];
- int bytes = is.read(buf, 0, 4096);
- is.close();
-
- assertEquals(data.length, bytes);
- for (int i = 0; i < bytes; i++) {
- assertEquals(data[i], buf[i]);
- }
- } finally {
- fs.delete(blobFile, false);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java b/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java
deleted file mode 100644
index c1ab7e5..0000000
--- a/src/test/com/cloudera/sqoop/lib/TestBooleanParser.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.lib;
-
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test the boolean parser.
- */
-public class TestBooleanParser {
-
- @Test
- public void testBoolParser() {
- assertTrue(BooleanParser.valueOf("true"));
- assertTrue(BooleanParser.valueOf("TRUE"));
- assertTrue(BooleanParser.valueOf("True"));
- assertTrue(BooleanParser.valueOf("t"));
- assertTrue(BooleanParser.valueOf("T"));
- assertTrue(BooleanParser.valueOf("on"));
- assertTrue(BooleanParser.valueOf("On"));
- assertTrue(BooleanParser.valueOf("ON"));
- assertTrue(BooleanParser.valueOf("yes"));
- assertTrue(BooleanParser.valueOf("yEs"));
- assertTrue(BooleanParser.valueOf("YES"));
- assertTrue(BooleanParser.valueOf("1"));
-
- assertFalse(BooleanParser.valueOf(null));
-
- assertFalse(BooleanParser.valueOf("no"));
- assertFalse(BooleanParser.valueOf("false"));
- assertFalse(BooleanParser.valueOf("FALSE"));
- assertFalse(BooleanParser.valueOf("0"));
- assertFalse(BooleanParser.valueOf("off"));
- assertFalse(BooleanParser.valueOf("OFF"));
- assertFalse(BooleanParser.valueOf("anything else in the world"));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestClobRef.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/lib/TestClobRef.java b/src/test/com/cloudera/sqoop/lib/TestClobRef.java
deleted file mode 100644
index 7efc3b6..0000000
--- a/src/test/com/cloudera/sqoop/lib/TestClobRef.java
+++ /dev/null
@@ -1,167 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.lib;
-
-import java.io.*;
-
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.testutil.CommonArgs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import com.cloudera.sqoop.io.LobFile;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test parsing of ClobRef objects.
- */
-public class TestClobRef {
-
- @Test
- public void testEmptyStr() {
- ClobRef r = ClobRef.parse("");
- assertFalse(r.isExternal());
- assertEquals("", r.toString());
- }
-
- @Test
- public void testInline() throws IOException {
- ClobRef r = ClobRef.parse("foo");
- assertFalse(r.isExternal());
- assertEquals("foo", r.toString());
-
- Reader reader = r.getDataStream(null, null);
- assertNotNull(reader);
- char [] buf = new char[4096];
- int chars = reader.read(buf, 0, 4096);
- reader.close();
-
- String str = new String(buf, 0, chars);
- assertEquals("foo", str);
- }
-
- @Test
- public void testEmptyFile() {
- ClobRef r = ClobRef.parse("externalLob()");
- assertFalse(r.isExternal());
- assertEquals("externalLob()", r.toString());
-
- r = ClobRef.parse("externalLob(lf,,0,0)");
- assertTrue(r.isExternal());
- assertEquals("externalLob(lf,,0,0)", r.toString());
- }
-
- @Test
- public void testInlineNearMatch() {
- ClobRef r = ClobRef.parse("externalLob(foo)bar");
- assertFalse(r.isExternal());
- assertEquals("externalLob(foo)bar", r.toString());
-
- r = ClobRef.parse("externalLob(foo)");
- assertFalse(r.isExternal());
- assertEquals("externalLob(foo)", r.getData());
-
- r = ClobRef.parse("externalLob(lf,foo)");
- assertFalse(r.isExternal());
- assertEquals("externalLob(lf,foo)", r.getData());
-
- r = ClobRef.parse("externalLob(lf,foo,1,2)x");
- assertFalse(r.isExternal());
- assertEquals("externalLob(lf,foo,1,2)x", r.getData());
- }
-
- @Test
- public void testExternal() throws IOException {
- final String DATA = "This is the clob data!";
- final String FILENAME = "clobdata";
-
- doExternalTest(DATA, FILENAME);
- }
-
- @Test
- public void testExternalSubdir() throws IOException {
- final String DATA = "This is the clob data!";
- final String FILENAME = "_lob/clobdata";
-
- try {
- doExternalTest(DATA, FILENAME);
- } finally {
- // remove dir we made.
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.getLocal(conf);
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
- Path lobDir = new Path(new Path(tmpDir), "_lob");
- fs.delete(lobDir, true);
- }
- }
-
- private void doExternalTest(final String data, final String filename)
- throws IOException {
-
- Configuration conf = new Configuration();
- if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- FileSystem fs = FileSystem.get(conf);
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
-
- Path tmpPath = new Path(tmpDir);
- Path clobFile = new Path(tmpPath, filename);
-
- // make any necessary parent dirs.
- Path clobParent = clobFile.getParent();
- if (!fs.exists(clobParent)) {
- fs.mkdirs(clobParent);
- }
-
- LobFile.Writer lw = LobFile.create(clobFile, conf, true);
- try {
- long off = lw.tell();
- long len = data.length();
- Writer w = lw.writeClobRecord(len);
- w.append(data);
- w.close();
- lw.close();
-
- String refString = "externalLob(lf," + filename
- + "," + off + "," + len + ")";
- ClobRef clob = ClobRef.parse(refString);
- assertTrue(clob.isExternal());
- assertEquals(refString, clob.toString());
- Reader r = clob.getDataStream(conf, tmpPath);
- assertNotNull(r);
-
- char [] buf = new char[4096];
- int chars = r.read(buf, 0, 4096);
- r.close();
-
- String str = new String(buf, 0, chars);
- assertEquals(data, str);
- } finally {
- fs.delete(clobFile, false);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java b/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java
deleted file mode 100644
index 1fc9cd2..0000000
--- a/src/test/com/cloudera/sqoop/lib/TestFieldFormatter.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.lib;
-
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNull;
-
-
-/**
- * Test that the field formatter works in a variety of configurations.
- */
-public class TestFieldFormatter {
-
- @Test
- public void testAllEmpty() {
- String result = FieldFormatter.escapeAndEnclose("",
- new DelimiterSet(DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR,
- DelimiterSet.NULL_CHAR, DelimiterSet.NULL_CHAR, false));
- assertEquals("", result);
- }
-
- @Test
- public void testNullArgs() {
- assertNull(FieldFormatter.escapeAndEnclose(null,
- new DelimiterSet('\"', DelimiterSet.NULL_CHAR, '\"', '\\', false)));
- }
-
- @Test
- public void testBasicStr() {
- String result = FieldFormatter.escapeAndEnclose("foo",
- DelimiterSet.DEFAULT_DELIMITERS);
- assertEquals("foo", result);
- }
-
- @Test
- public void testEscapeSlash() {
- String result = FieldFormatter.escapeAndEnclose("foo\\bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("foo\\\\bar", result);
- }
-
- @Test
- public void testMustEnclose() {
- String result = FieldFormatter.escapeAndEnclose("foo",
- new DelimiterSet(',', '\n', '\"', DelimiterSet.NULL_CHAR, true));
- assertEquals("\"foo\"", result);
- }
-
- @Test
- public void testEncloseComma1() {
- String result = FieldFormatter.escapeAndEnclose("foo,bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("\"foo,bar\"", result);
- }
-
- @Test
- public void testEncloseComma2() {
- String result = FieldFormatter.escapeAndEnclose("foo,bar",
- new DelimiterSet(',', ',', '\"', '\\', false));
- assertEquals("\"foo,bar\"", result);
- }
-
- @Test
- public void testNoNeedToEnclose() {
- String result = FieldFormatter.escapeAndEnclose(
- "just another string",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("just another string", result);
- }
-
- @Test
- public void testCannotEnclose() {
- // Can't enclose because encloser is nul.
- // This should escape the comma instead.
- String result = FieldFormatter.escapeAndEnclose("foo,bar",
- new DelimiterSet(',', '\n', DelimiterSet.NULL_CHAR, '\\', false));
-
- assertEquals("foo\\,bar", result);
- }
-
- @Test
- public void testEmptyCharToEscapeString() {
- // test what happens when the escape char is null. It should encode the
- // null char.
-
- char nul = DelimiterSet.NULL_CHAR;
- String s = "" + nul;
- assertEquals("\000", s);
- }
-
- @Test
- public void testEscapeCentralQuote() {
- String result = FieldFormatter.escapeAndEnclose("foo\"bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("foo\\\"bar", result);
- }
-
- @Test
- public void testEscapeMultiCentralQuote() {
- String result = FieldFormatter.escapeAndEnclose("foo\"\"bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("foo\\\"\\\"bar", result);
- }
-
- @Test
- public void testDoubleEscape() {
- String result = FieldFormatter.escapeAndEnclose("foo\\\"bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("foo\\\\\\\"bar", result);
- }
-
- @Test
- public void testReverseEscape() {
- String result = FieldFormatter.escapeAndEnclose("foo\"\\bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("foo\\\"\\\\bar", result);
- }
-
- @Test
- public void testQuotedEncloser() {
- String result = FieldFormatter.escapeAndEnclose("foo\",bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("\"foo\\\",bar\"", result);
- }
-
- @Test
- public void testQuotedEscape() {
- String result = FieldFormatter.escapeAndEnclose("foo\\,bar",
- new DelimiterSet(',', '\n', '\"', '\\', false));
- assertEquals("\"foo\\\\,bar\"", result);
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java b/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
deleted file mode 100644
index c55f14b..0000000
--- a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.lib;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.Reader;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import com.cloudera.sqoop.testutil.BaseSqoopTestCase;
-import com.cloudera.sqoop.testutil.CommonArgs;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-
-import com.cloudera.sqoop.testutil.MockResultSet;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-/**
- * Test deserialization of ClobRef and BlobRef fields.
- */
-public class TestLargeObjectLoader {
-
- protected Configuration conf;
- protected LargeObjectLoader loader;
- protected Path outDir;
-
- @Before
- public void setUp() throws IOException, InterruptedException {
- conf = new Configuration();
- if (!BaseSqoopTestCase.isOnPhysicalCluster()) {
- conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS);
- }
- String tmpDir = System.getProperty("test.build.data", "/tmp/");
- this.outDir = new Path(System.getProperty("java.io.tmpdir"));
- FileSystem fs = FileSystem.get(conf);
- if (fs.exists(outDir)) {
- fs.delete(outDir, true);
- }
- fs.mkdirs(outDir);
-
- loader = new LargeObjectLoader(conf, outDir);
- }
-
- @Test
- public void testReadClobRef()
- throws IOException, InterruptedException, SQLException {
- // This should give us an inline CLOB.
- ResultSet resultSet = new MockResultSet();
- ClobRef clob = loader.readClobRef(0, resultSet);
- assertNotNull(clob);
- assertFalse(clob.isExternal());
- assertEquals(MockResultSet.CLOB_DATA, clob.toString());
-
- // LOBs bigger than 4 bytes are now external.
- conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4);
- clob = loader.readClobRef(0, resultSet);
- assertNotNull(clob);
- assertTrue(clob.isExternal());
- loader.close();
- Reader r = clob.getDataStream(conf, outDir);
- char [] buf = new char[4096];
- int chars = r.read(buf, 0, 4096);
- r.close();
- String str = new String(buf, 0, chars);
- assertEquals(MockResultSet.CLOB_DATA, str);
- }
-
- @Test
- public void testReadBlobRef()
- throws IOException, InterruptedException, SQLException {
- // This should give us an inline BLOB.
- ResultSet resultSet = new MockResultSet();
- BlobRef blob = loader.readBlobRef(0, resultSet);
- assertNotNull(blob);
- assertFalse(blob.isExternal());
- byte [] data = blob.getData();
- byte [] blobData = MockResultSet.blobData();
- assertEquals(blobData.length, data.length);
- for (int i = 0; i < data.length; i++) {
- assertEquals(blobData[i], data[i]);
- }
-
- // LOBs bigger than 4 bytes are now external.
- conf.setLong(LargeObjectLoader.MAX_INLINE_LOB_LEN_KEY, 4);
- blob = loader.readBlobRef(0, resultSet);
- assertNotNull(blob);
- assertTrue(blob.isExternal());
- loader.close();
- InputStream is = blob.getDataStream(conf, outDir);
- byte [] buf = new byte[4096];
- int bytes = is.read(buf, 0, 4096);
- is.close();
-
- assertEquals(blobData.length, bytes);
- for (int i = 0; i < bytes; i++) {
- assertEquals(blobData[i], buf[i]);
- }
- }
-}
-
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/lib/TestRecordParser.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/lib/TestRecordParser.java b/src/test/com/cloudera/sqoop/lib/TestRecordParser.java
deleted file mode 100644
index d964cef..0000000
--- a/src/test/com/cloudera/sqoop/lib/TestRecordParser.java
+++ /dev/null
@@ -1,450 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.lib;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.ExpectedException;
-
-import static org.junit.Assert.fail;
-
-
-/**
- * Test that the record parser works in a variety of configurations.
- */
-public class TestRecordParser {
-
- @Rule
- public ExpectedException thrown = ExpectedException.none();
-
- private void assertListsEqual(String msg, List<String> expected,
- List<String> actual) {
- if (expected == null && actual != null) {
- if (null == msg) {
- msg = "expected null list";
- }
-
- fail(msg);
- } else if (expected != null && actual == null) {
- if (null == msg) {
- msg = "expected non-null list";
- }
-
- fail(msg);
- } else if (expected == null && actual == null) {
- return; // ok. Both null; nothing to do.
- }
-
- assert(null != expected);
- assert(null != actual);
-
- int expectedLen = expected.size();
- int actualLen = actual.size();
-
- if (expectedLen != actualLen) {
- if (null == msg) {
- msg = "Expected list of length " + expectedLen
- + "; got " + actualLen;
- }
-
- fail(msg);
- }
-
- // Check the list contents.
- for (int i = 0; i < expectedLen; i++) {
- String expectedElem = expected.get(i);
- String actualElem = actual.get(i);
-
- if (expectedElem == null) {
- if (actualElem != null) {
- if (null == msg) {
- msg = "Expected null element at position " + i
- + "; got [" + actualElem + "]";
- }
-
- fail(msg);
- }
- } else if (!expectedElem.equals(actualElem)) {
- if (null == msg) {
- msg = "Expected [" + expectedElem + "] at position " + i
- + "; got [" + actualElem + "]";
- }
-
- fail(msg);
- }
- }
- }
-
- private List<String> list(String [] items) {
-
- if (null == items) {
- return null;
- }
-
- ArrayList<String> asList = new ArrayList<String>();
- for (int i = 0; i < items.length; i++) {
- asList.add(items[i]);
- }
-
- return asList;
- }
-
- @Test
- public void testEmptyLine() throws RecordParser.ParseError {
- // an empty line should return no fields.
-
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { };
- assertListsEqual(null, list(strings), parser.parseRecord(""));
- }
-
- @Test
- public void testJustEOR() throws RecordParser.ParseError {
- // a line with just a newline char should return a single zero-length field.
-
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "" };
- assertListsEqual(null, list(strings), parser.parseRecord("\n"));
- }
-
- @Test
- public void testOneField() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the field" };
- assertListsEqual(null, list(strings), parser.parseRecord("the field"));
- }
-
- @Test
- public void testOneField2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the field" };
- assertListsEqual(null, list(strings), parser.parseRecord("the field\n"));
- }
-
- @Test
- public void testQuotedField1() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the field" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the field\"\n"));
- }
-
- @Test
- public void testQuotedField2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the field" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the field\""));
- }
-
- @Test
- public void testQuotedField3() throws RecordParser.ParseError {
- // quoted containing EOF
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the ,field" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the ,field\""));
- }
-
- @Test
- public void testQuotedField4() throws RecordParser.ParseError {
- // quoted containing multiple EOFs
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the ,,field" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the ,,field\""));
- }
-
- @Test
- public void testQuotedField5() throws RecordParser.ParseError {
- // quoted containing EOF and EOR
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the ,\nfield" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the ,\nfield\""));
- }
-
- @Test
- public void testQuotedField6() throws RecordParser.ParseError {
- // quoted containing EOR
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the \nfield" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the \nfield\""));
- }
-
- @Test
- public void testQuotedField7() throws RecordParser.ParseError {
- // quoted containing multiple EORs
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the \n\nfield" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the \n\nfield\""));
- }
-
- @Test
- public void testQuotedField8() throws RecordParser.ParseError {
- // quoted containing escaped quoted char
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the \"field" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"the \\\"field\""));
- }
-
- @Test
- public void testUnquotedEscape1() throws RecordParser.ParseError {
- // field without quotes with an escaped EOF char.
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the ,field" };
- assertListsEqual(null, list(strings), parser.parseRecord("the \\,field"));
- }
-
- @Test
- public void testUnquotedEscape2() throws RecordParser.ParseError {
- // field without quotes with an escaped escape char.
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "the \\field" };
- assertListsEqual(null, list(strings), parser.parseRecord("the \\\\field"));
- }
-
- @Test
- public void testTwoFields1() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings), parser.parseRecord("field1,field2"));
- }
-
- @Test
- public void testTwoFields2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("field1,field2\n"));
- }
-
- @Test
- public void testTwoFields3() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"field1\",field2\n"));
- }
-
- @Test
- public void testTwoFields4() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("field1,\"field2\"\n"));
- }
-
- @Test
- public void testTwoFields5() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("field1,\"field2\""));
- }
-
- @Test
- public void testRequiredQuotes0() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', true));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"field1\",\"field2\"\n"));
- }
-
- @Test
- public void testRequiredQuotes1() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', true));
- String [] strings = { "field1", "field2" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("\"field1\",\"field2\""));
- }
-
- @Test
- public void testRequiredQuotes2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', true));
-
- thrown.expect(RecordParser.ParseError.class);
- thrown.reportMissingExceptionWithMessage("Expected parse error for required quotes");
- parser.parseRecord("\"field1\",field2");
- }
-
- @Test
- public void testRequiredQuotes3() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', true));
-
- thrown.expect(RecordParser.ParseError.class);
- thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes");
- parser.parseRecord("field1,\"field2\"");
- }
-
- @Test
- public void testRequiredQuotes4() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', true));
-
- thrown.expect(RecordParser.ParseError.class);
- thrown.reportMissingExceptionWithMessage("Expected ParseError for required quotes");
- parser.parseRecord("field1,\"field2\"\n");
- }
-
- @Test
- public void testNull() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', true));
- String input = null;
-
- thrown.expect(RecordParser.ParseError.class);
- thrown.reportMissingExceptionWithMessage("Expected ParseError for null string");
- parser.parseRecord(input);
- }
-
-
- @Test
- public void testEmptyFields1() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "", ""};
- assertListsEqual(null, list(strings), parser.parseRecord(","));
- }
-
- @Test
- public void testEmptyFields2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "", "" };
- assertListsEqual(null, list(strings), parser.parseRecord(",\n"));
- }
-
- @Test
- public void testEmptyFields3() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "", "", "" };
- assertListsEqual(null, list(strings), parser.parseRecord(",,\n"));
- }
-
- @Test
- public void testEmptyFields4() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "", "foo", "" };
- assertListsEqual(null, list(strings), parser.parseRecord(",foo,\n"));
- }
-
- @Test
- public void testEmptyFields5() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "", "foo", "" };
- assertListsEqual(null, list(strings), parser.parseRecord(",foo,"));
- }
-
- @Test
- public void testEmptyFields6() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "foo", "" };
- assertListsEqual(null, list(strings), parser.parseRecord("foo,"));
- }
-
- @Test
- public void testTrailingText() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "foo", "bar" };
- assertListsEqual(null, list(strings), parser.parseRecord("foo,bar\nbaz"));
- }
-
- @Test
- public void testTrailingText2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "" };
- assertListsEqual(null, list(strings), parser.parseRecord("\nbaz"));
- }
-
- @Test
- public void testLeadingEscape() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', '\n', '\"', '\\', false));
- String [] strings = { "\nbaz" };
- assertListsEqual(null, list(strings), parser.parseRecord("\\\nbaz"));
- }
-
- @Test
- public void testEofIsEor() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', ',', '\"', '\\', false));
- String [] strings = { "three", "different", "fields" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("three,different,fields"));
- }
-
- @Test
- public void testEofIsEor2() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', ',', '\"', '\\', false));
- String [] strings = { "three", "different", "fields" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("three,\"different\",fields"));
- }
-
- @Test
- public void testRepeatedParse() throws RecordParser.ParseError {
- RecordParser parser = new RecordParser(
- new DelimiterSet(',', ',', '\"', '\\', false));
- String [] strings = { "three", "different", "fields" };
- assertListsEqual(null, list(strings),
- parser.parseRecord("three,\"different\",fields"));
-
- String [] strings2 = { "foo", "bar" };
- assertListsEqual(null, list(strings2),
- parser.parseRecord("foo,\"bar\""));
- }
-
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/6984a36c/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java b/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java
deleted file mode 100644
index 36aa821..0000000
--- a/src/test/com/cloudera/sqoop/manager/CubridManagerExportTest.java
+++ /dev/null
@@ -1,306 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package com.cloudera.sqoop.manager;
-
-import java.io.BufferedWriter;
-import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.Writer;
-import java.sql.Connection;
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.sqoop.manager.CubridManager;
-import org.apache.sqoop.manager.cubrid.CubridTestUtils;
-import org.junit.After;
-import org.junit.Before;
-
-import com.cloudera.sqoop.SqoopOptions;
-import com.cloudera.sqoop.TestExport;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.fail;
-
-/**
- * Test the CubridManager implementation.
- *
- * This uses JDBC to export data from HDFS to an Cubrid database.
- *
- * Since this requires an Cubrid installation, this class is named in such a way
- * that Sqoop's default QA process does not run it. You need to run this
- * manually with -Dtestcase=CubridManagerExportTest.
- *
- * You need to put Cubrid JDBC driver library (JDBC-9.2.2.0003-cubrid.jar) in a
- * location where Sqoop will be able to access it (since this library cannot be
- * checked into Apache's tree for licensing reasons).
- *
- * To set up your test environment:
- * Install Cubrid 9.2.2
- * ref:http://www.cubrid.org/wiki_tutorials/entry/installing-cubrid-on-linux-using-shell-and-rpm
- * Create a database SQOOPCUBRIDTEST
- * $cubrid createdb SQOOPCUBRIDTEST en_us.utf8
- * Start cubrid and database
- * $cubrid service start
- * $cubrid server start SQOOPCUBRIDTEST
- * Create a login SQOOPUSER with password PASSWORD and grant all
- * $csql -u dba SQOOPCUBRIDTEST
- * csql>CREATE USER SQOOPUSER password 'PASSWORD';
- */
-public class CubridManagerExportTest extends TestExport {
-
- public static final Log LOG = LogFactory.getLog(
- CubridManagerExportTest.class.getName());
-
- static final String TABLE_PREFIX = "EXPORT_CUBRID_";
-
- // instance variables populated during setUp, used during tests.
- private CubridManager manager;
- private Connection conn;
-
- @Override
- protected Connection getConnection() {
- return conn;
- }
-
- @Override
- protected boolean useHsqldbTestServer() {
- return false;
- }
-
- @Override
- protected String getConnectString() {
- return CubridTestUtils.getConnectString();
- }
-
- @Override
- protected String getTablePrefix() {
- return TABLE_PREFIX;
- }
-
- @Override
- protected String getDropTableStatement(String tableName) {
- return "DROP TABLE IF EXISTS " + tableName;
- }
-
- /**
- * Cubrid could not support --staging-table, Diable this test case.
- */
- @Override
- @Test
- public void testMultiTransactionWithStaging() throws IOException,
- SQLException {
- return;
- }
-
- /**
- * Cubrid could not support --staging-table, Diable this test case.
- */
- @Override
- @Test
- public void testMultiMapTextExportWithStaging() throws IOException,
- SQLException {
- return;
- }
-
- public void createTableAndPopulateData(String table) {
- String fulltableName = manager.escapeTableName(table);
-
- Statement stmt = null;
-
- // Drop the existing table, if there is one.
- try {
- conn = manager.getConnection();
- stmt = conn.createStatement();
- stmt.execute("DROP TABLE IF EXISTS " + fulltableName);
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.info("Table was not dropped: " + sqlE.getMessage());
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn("Exception while closing stmt", ex);
- }
- }
-
- // Create and populate table
- try {
- conn = manager.getConnection();
- conn.setAutoCommit(false);
- stmt = conn.createStatement();
-
- // create the database table and populate it with data.
- stmt.executeUpdate("CREATE TABLE "
- + fulltableName + " ("
- + "id INT NOT NULL, "
- + "name VARCHAR(24) NOT NULL, "
- + "salary FLOAT, " + "dept VARCHAR(32), "
- + "PRIMARY KEY (id))");
- conn.commit();
- } catch (SQLException sqlE) {
- LOG.error("Encountered SQL Exception: ", sqlE);
- sqlE.printStackTrace();
- fail("SQLException when running test setUp(): " + sqlE);
- } finally {
- try {
- if (null != stmt) {
- stmt.close();
- }
- } catch (Exception ex) {
- LOG.warn(
- "Exception while closing connection/stmt", ex);
- }
- }
- }
-
- @Before
- public void setUp() {
- super.setUp();
-
- SqoopOptions options = new SqoopOptions(
- CubridTestUtils.getConnectString(),
- getTableName());
- options.setUsername(CubridTestUtils.getCurrentUser());
- options.setPassword(CubridTestUtils.getPassword());
- this.manager = new CubridManager(options);
- try {
- this.conn = manager.getConnection();
- this.conn.setAutoCommit(false);
-
- } catch (SQLException sqlE) {
- LOG.error(StringUtils.stringifyException(sqlE));
- fail("Failed with sql exception in setup: " + sqlE);
- }
- }
-
- @After
- public void tearDown() {
- super.tearDown();
- try {
- conn.close();
- manager.close();
- } catch (SQLException sqlE) {
- LOG.error("Got SQLException: " + sqlE.toString());
- fail("Got SQLException: " + sqlE.toString());
- }
- }
-
- @Override
- protected String[] getCodeGenArgv(String... extraArgs) {
- String[] moreArgs = new String[extraArgs.length + 4];
- int i = 0;
- for (i = 0; i < extraArgs.length; i++) {
- moreArgs[i] = extraArgs[i];
- }
-
- // Add username and password args.
- moreArgs[i++] = "--username";
- moreArgs[i++] = CubridTestUtils.getCurrentUser();
- moreArgs[i++] = "--password";
- moreArgs[i++] = CubridTestUtils.getPassword();
-
- return super.getCodeGenArgv(moreArgs);
- }
-
- @Override
- protected String[] getArgv(boolean includeHadoopFlags,
- int rowsPerStatement,
- int statementsPerTx, String... additionalArgv) {
-
- String[] subArgv = newStrArray(additionalArgv, "--username",
- CubridTestUtils.getCurrentUser(), "--password",
- CubridTestUtils.getPassword());
- return super.getArgv(includeHadoopFlags, rowsPerStatement,
- statementsPerTx, subArgv);
- }
-
- protected void createTestFile(String filename,
- String[] lines)
- throws IOException {
- File testdir = new File(getWarehouseDir());
- if (!testdir.exists()) {
- testdir.mkdirs();
- }
- File file = new File(getWarehouseDir() + "/" + filename);
- Writer output = new BufferedWriter(new FileWriter(file));
- for (String line : lines) {
- output.write(line);
- output.write("\n");
- }
- output.close();
- }
-
- public static void assertRowCount(long expected,
- String tableName,
- Connection connection) {
- Statement stmt = null;
- ResultSet rs = null;
- try {
- stmt = connection.createStatement();
- rs = stmt.executeQuery("SELECT count(*) FROM "
- + tableName);
- rs.next();
- assertEquals(expected, rs.getLong(1));
- } catch (SQLException e) {
- LOG.error("Can't verify number of rows", e);
- fail();
- } finally {
- try {
- connection.commit();
- if (stmt != null) {
- stmt.close();
- }
- if (rs != null) {
- rs.close();
- }
- } catch (SQLException ex) {
- LOG.info("Ignored exception in finally block.");
- }
- }
- }
-
- public String escapeTableOrSchemaName(String tableName) {
- return "\"" + tableName + "\"";
- }
-
- /** Make sure mixed update/insert export work correctly. */
- @Test
- public void testUpsertTextExport() throws IOException, SQLException {
- final int TOTAL_RECORDS = 10;
- createTextFile(0, TOTAL_RECORDS, false);
- createTable();
- // first time will be insert.
- runExport(getArgv(true, 10, 10,
- newStrArray(null, "--update-key", "id",
- "--update-mode", "allowinsert")));
- // second time will be update.
- runExport(getArgv(true, 10, 10,
- newStrArray(null, "--update-key", "id",
- "--update-mode", "allowinsert")));
- verifyExport(TOTAL_RECORDS);
- }
-}