You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ka...@apache.org on 2013/06/24 00:45:18 UTC
git commit: SQOOP-1087: Sqoop2: Integration: Abstract common
functionality into src module
Updated Branches:
refs/heads/sqoop2 fe54d473a -> f980e90fc
SQOOP-1087: Sqoop2: Integration: Abstract common functionality into src module
(Jarek Jarcec Cecho via Kate Ting)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/f980e90f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/f980e90f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/f980e90f
Branch: refs/heads/sqoop2
Commit: f980e90fc9570b01305166b19d7381c2051bc49e
Parents: fe54d47
Author: Kate Ting <ka...@apache.org>
Authored: Sun Jun 23 18:43:50 2013 -0400
Committer: Kate Ting <ka...@apache.org>
Committed: Sun Jun 23 18:43:50 2013 -0400
----------------------------------------------------------------------
test/pom.xml | 1 -
.../apache/sqoop/test/asserts/HdfsAsserts.java | 83 ++++++++
.../sqoop/test/asserts/ProviderAsserts.java | 74 +++++++
.../java/org/apache/sqoop/test/data/Cities.java | 53 +++++
.../org/apache/sqoop/test/data/DataSet.java | 66 ++++++
.../sqoop/test/testcases/ConnectorTestCase.java | 177 ++++++++++++++++
.../sqoop/test/testcases/TomcatTestCase.java | 146 +++++++++++++
.../org/apache/sqoop/test/utils/HdfsUtils.java | 69 ++++++
.../sqoop/integration/TomcatTestCase.java | 197 -----------------
.../connector/ConnectorTestCase.java | 209 -------------------
.../connector/jdbc/generic/TableExportTest.java | 10 +-
.../connector/jdbc/generic/TableImportTest.java | 2 +-
.../sqoop/integration/server/VersionTest.java | 2 +-
13 files changed, 675 insertions(+), 414 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/pom.xml
----------------------------------------------------------------------
diff --git a/test/pom.xml b/test/pom.xml
index b7ea1ed..8001fce 100644
--- a/test/pom.xml
+++ b/test/pom.xml
@@ -33,7 +33,6 @@ limitations under the License.
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
- <scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
new file mode 100644
index 0000000..056e612
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/asserts/HdfsAsserts.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.asserts;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.apache.sqoop.test.utils.HdfsUtils;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import static org.junit.Assert.fail;
+
+/**
+ * Assert methods suitable for checking HDFS files and directories.
+ *
+ * TODO: This module will require clean up to work on MiniCluster/Real cluster.
+ */
+public class HdfsAsserts {
+
+ private static final Logger LOG = Logger.getLogger(HdfsAsserts.class);
+
+ /**
+ * Verify that mapreduce output (across all files) is as expected.
+ *
+ * @param directory Mapreduce output directory
+ * @param lines Expected lines
+ * @throws IOException
+ */
+ public static void assertMapreduceOutput(String directory, String... lines) throws IOException {
+ Set<String> setLines = new HashSet<String>(Arrays.asList(lines));
+ List<String> notFound = new LinkedList<String>();
+
+ String []files = HdfsUtils.getOutputMapreduceFiles(directory);
+
+ for(String file : files) {
+ String filePath = directory + "/" + file;
+ BufferedReader br = new BufferedReader(new FileReader((filePath)));
+
+ String line;
+ while ((line = br.readLine()) != null) {
+ if (!setLines.remove(line)) {
+ notFound.add(line);
+ }
+ }
+ br.close();
+ }
+
+ if(!setLines.isEmpty() || !notFound.isEmpty()) {
+ LOG.error("Expected lines that weren't present in the files:");
+ LOG.error("\t" + StringUtils.join(setLines, "\n\t"));
+ LOG.error("Extra lines in files that weren't expected:");
+ LOG.error("\t" + StringUtils.join(notFound, "\n\t"));
+ fail("Output do not match expectations.");
+ }
+ }
+
+
+ private HdfsAsserts() {
+ // Instantiation is prohibited
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java
new file mode 100644
index 0000000..364f86d
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/asserts/ProviderAsserts.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.asserts;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+import org.apache.log4j.Logger;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Database provider related asserts.
+ */
+public class ProviderAsserts {
+
+ private static final Logger LOG = Logger.getLogger(ProviderAsserts.class);
+
+ /**
+ * Assert row in the table.
+ *
+ * @param provider Provider that should be used to query the database
+ * @param tableName Table name
+ * @param conditions Conditions for identifying the row
+ * @param values Values that should be present in the table
+ */
+ public static void assertRow(DatabaseProvider provider, String tableName, Object []conditions, Object ...values) {
+ ResultSet rs = null;
+ try {
+ rs = provider.getRows(tableName, conditions);
+
+ if(! rs.next()) {
+ fail("No rows found.");
+ }
+
+ int i = 1;
+ for(Object expectedValue : values) {
+ Object actualValue = rs.getObject(i);
+ assertEquals("Columns do not match on position: " + i, expectedValue, actualValue);
+ i++;
+ }
+
+ if(rs.next()) {
+ fail("Found more than one row.");
+ }
+ } catch (SQLException e) {
+ LOG.error("Unexpected SQLException", e);
+ fail("Unexpected SQLException: " + e);
+ } finally {
+ provider.closeResultSetWithStatement(rs);
+ }
+ }
+
+ private ProviderAsserts() {
+ // Instantiation is prohibited
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/data/Cities.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/data/Cities.java b/test/src/main/java/org/apache/sqoop/test/data/Cities.java
new file mode 100644
index 0000000..fb9be13
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/data/Cities.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.data;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+
+/**
+ * Simple listing of few world's cities to do basic sanity tests.
+ */
+public class Cities extends DataSet {
+
+ public Cities(DatabaseProvider provider, String tableBaseName) {
+ super(provider, tableBaseName);
+ }
+
+ @Override
+ public DataSet createTables() {
+ provider.createTable(
+ tableBaseName,
+ "id",
+ "id", "int",
+ "country", "varchar(50)",
+ "city", "varchar(50)"
+ );
+
+ return this;
+ }
+
+ @Override
+ public DataSet loadBasicData() {
+ provider.insertRow(tableBaseName, 1, "USA", "San Francisco");
+ provider.insertRow(tableBaseName, 2, "USA", "Sunnyvale");
+ provider.insertRow(tableBaseName, 3, "Czech Republic", "Brno");
+ provider.insertRow(tableBaseName, 4, "USA", "Palo Alto");
+
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/data/DataSet.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/data/DataSet.java b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java
new file mode 100644
index 0000000..6999408
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/data/DataSet.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.data;
+
+import org.apache.sqoop.test.db.DatabaseProvider;
+
+/**
+ * Abstract class for basic testing data sets.
+ *
+ * Each data set provides couple of generic methods that can be used to set up
+ * the tables and load example data.
+ */
+public abstract class DataSet {
+
+ /**
+ * Database provider that will be used to populate the data.
+ */
+ protected DatabaseProvider provider;
+
+ /**
+ * Base name for created tables.
+ */
+ protected String tableBaseName;
+
+ public DataSet(DatabaseProvider provider, String tableBaseName) {
+ setProvider(provider);
+ setTableBaseName(tableBaseName);
+ }
+
+ public DataSet setProvider(DatabaseProvider provider) {
+ this.provider = provider;
+ return this;
+ }
+
+ public DataSet setTableBaseName(String tableBaseName) {
+ this.tableBaseName = tableBaseName;
+ return this;
+ }
+
+ /**
+ * Crate all tables that this testing data set might need.
+ */
+ public abstract DataSet createTables();
+
+ /**
+ * Load basic data.
+ *
+ * Basic data set should be small (around 10 rows) without any specialities.
+ */
+ public abstract DataSet loadBasicData();
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
new file mode 100644
index 0000000..6aeadd4
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/ConnectorTestCase.java
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testcases;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.framework.configuration.OutputFormat;
+import org.apache.sqoop.framework.configuration.StorageType;
+import org.apache.sqoop.model.MConnection;
+import org.apache.sqoop.model.MFormList;
+import org.apache.sqoop.model.MJob;
+import org.apache.sqoop.model.MPersistableEntity;
+import org.apache.sqoop.test.asserts.ProviderAsserts;
+import org.apache.sqoop.test.data.Cities;
+import org.apache.sqoop.test.db.DatabaseProvider;
+import org.apache.sqoop.test.db.DatabaseProviderFactory;
+import org.apache.sqoop.validation.Status;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotSame;
+
+/**
+ * Base test case suitable for connector testing.
+ *
+ * In addition to pure Tomcat based test case it will also create and initialize
+ * the database provider prior every test execution.
+ */
+abstract public class ConnectorTestCase extends TomcatTestCase {
+
+ private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
+
+ protected static DatabaseProvider provider;
+
+ @BeforeClass
+ public static void startProvider() throws Exception {
+ provider = DatabaseProviderFactory.getProvider(System.getProperties());
+ LOG.info("Starting database provider: " + provider.getClass().getName());
+ provider.start();
+ }
+
+ @AfterClass
+ public static void stopProvider() {
+ LOG.info("Stopping database provider: " + provider.getClass().getName());
+ provider.stop();
+ }
+
+ public String getTableName() {
+ return getClass().getSimpleName();
+ }
+
+ protected void createTable(String primaryKey, String ...columns) {
+ provider.createTable(getTableName(), primaryKey, columns);
+ }
+
+ protected void dropTable() {
+ provider.dropTable(getTableName());
+ }
+
+ protected void insertRow(Object ...values) {
+ provider.insertRow(getTableName(), values);
+ }
+
+ protected long rowCount() {
+ return provider.rowCount(getTableName());
+ }
+
+ /**
+ * Fill connection form based on currently active provider.
+ *
+ * @param connection MConnection object to fill
+ */
+ protected void fillConnectionForm(MConnection connection) {
+ MFormList forms = connection.getConnectorPart();
+ forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
+ forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
+ forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
+ forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
+ }
+
+ /**
+ * Fill output form with specific storage and output type. Mapreduce output directory
+ * will be set to default test value.
+ *
+ * @param job MJOb object to fill
+ * @param storage Storage type that should be set
+ * @param output Output type that should be set
+ */
+ protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) {
+ MFormList forms = job.getFrameworkPart();
+ forms.getEnumInput("output.storageType").setValue(storage);
+ forms.getEnumInput("output.outputFormat").setValue(output);
+ forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
+ }
+
+ /**
+ * Fill input form. Mapreduce input directory will be set to default test value.
+ *
+ * @param job MJOb object to fill
+ */
+ protected void fillInputForm(MJob job) {
+ MFormList forms = job.getFrameworkPart();
+ forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory());
+ }
+
+ /**
+ * Create table cities.
+ */
+ protected void createTableCities() {
+ new Cities(provider, getTableName()).createTables();
+ }
+
+ /**
+ * Create table cities and load few rows.
+ */
+ protected void createAndLoadTableCities() {
+ new Cities(provider, getTableName()).createTables().loadBasicData();
+ }
+
+ /**
+ * Assert row in testing table.
+ *
+ * @param conditions Conditions in form that are expected by the database provider
+ * @param values Values that are expected in the table (with corresponding types)
+ */
+ protected void assertRow(Object []conditions, Object ...values) {
+ ProviderAsserts.assertRow(provider, getTableName(), conditions, values);
+ }
+
+ /**
+ * Assert row in table "cities".
+ *
+ * @param values Values that are expected
+ */
+ protected void assertRowInCities(Object... values) {
+ assertRow(new Object[]{"id", values[0]}, values);
+ }
+
+ /**
+ * Create connection.
+ *
+ * With asserts to make sure that it was created correctly.
+ *
+ * @param connection
+ */
+ protected void createConnection(MConnection connection) {
+ assertEquals(Status.FINE, getClient().createConnection(connection));
+ assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
+ }
+
+ /**
+ * Create job.
+ *
+ * With asserts to make sure that it was created correctly.
+ *
+ * @param job
+ */
+ protected void createJob(MJob job) {
+ assertEquals(Status.FINE, getClient().createJob(job));
+ assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
new file mode 100644
index 0000000..ca77e64
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/testcases/TomcatTestCase.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.testcases;
+
+import org.apache.log4j.Logger;
+import org.apache.sqoop.client.SqoopClient;
+import org.apache.sqoop.test.asserts.HdfsAsserts;
+import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
+import org.apache.sqoop.test.utils.HdfsUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.TestName;
+
+import java.io.IOException;
+
+/**
+ * Basic test case that will bootstrap Sqoop server running in external Tomcat
+ * process.
+ */
+abstract public class TomcatTestCase {
+
+ private static final Logger LOG = Logger.getLogger(TomcatTestCase.class);
+
+ @Rule public TestName name = new TestName();
+
+ /**
+ * Temporary base path that will be used for tests.
+ *
+ * By default we will take a look for sqoop.integration.tmpdir property that is
+ * filled up by maven. If the test is not started from maven (IDE) we will
+ * pick up configured java.io.tmpdir value. The last results is /tmp/ directory
+ * in case that no property is set.
+ */
+ private static final String TMP_PATH_BASE =
+ System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests/";
+
+ /**
+ * Temporary directory that will be used by the test.
+ *
+ * We will take TMP_PATH_BASE and append two subdirectories. First will be named
+ * after fully qualified class name of current test class, second directory will
+ * be named after current test method name. For example:
+ *
+ * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/
+ */
+ private String tmpPath;
+
+ /**
+ * Tomcat based Sqoop mini cluster
+ */
+ private TomcatSqoopMiniCluster cluster;
+
+ /**
+ * Sqoop client API.
+ */
+ private SqoopClient client;
+
+ @Before
+ public void startServer() throws Exception {
+ // Set up the temporary path
+ tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/";
+
+ // Set up and start server
+ cluster = new TomcatSqoopMiniCluster(getTemporaryPath());
+ cluster.start();
+
+ // Initialize Sqoop Client API
+ client = new SqoopClient(getServerUrl());
+ }
+
+ @After
+ public void stopServer() throws Exception {
+ cluster.stop();
+ }
+
+ /**
+ * Return SqoopClient configured to talk to testing server.
+ *
+ * @return
+ */
+ public SqoopClient getClient() {
+ return client;
+ }
+
+ public String getTemporaryPath() {
+ return tmpPath;
+ }
+
+ /**
+ * Return testing server URL
+ *
+ * @return
+ */
+ public String getServerUrl() {
+ return cluster.getServerUrl();
+ }
+
+ /**
+ * Get input/output directory for mapreduce job.
+ *
+ * @return
+ */
+ public String getMapreduceDirectory() {
+ return getTemporaryPath() + "/mapreduce-job-io";
+ }
+
+ /**
+ * Assert that mapreduce has generated following lines.
+ *
+ * As the lines can be spread between multiple files the ordering do not make
+ * a difference.
+ *
+ * @param lines
+ * @throws IOException
+ */
+ protected void assertMapreduceOutput(String... lines) throws IOException {
+ HdfsAsserts.assertMapreduceOutput(getMapreduceDirectory(), lines);
+ }
+
+ /**
+ * Create mapreduce input file with specified content.
+ *
+ * @param filename Input file name
+ * @param lines Individual lines that should be written into the file
+ * @throws IOException
+ */
+ protected void createInputMapreduceFile(String filename, String...lines) throws IOException {
+ HdfsUtils.createFile(getMapreduceDirectory(), filename, lines);
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
----------------------------------------------------------------------
diff --git a/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
new file mode 100644
index 0000000..95dd177
--- /dev/null
+++ b/test/src/main/java/org/apache/sqoop/test/utils/HdfsUtils.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.test.utils;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.Arrays;
+
+/**
+ * Handy utilities to work with HDFS
+ *
+ * TODO: This module will require clean up to work on MiniCluster/Real cluster.
+ */
+public class HdfsUtils {
+
+ private static final Logger LOG = Logger.getLogger(HdfsUtils.class);
+
+ /**
+ * Get list of mapreduce output files from given directory.
+ *
+ * @param directory Directory to be searched for files generated by MR
+ * @return
+ */
+ public static String [] getOutputMapreduceFiles(String directory) {
+ File dir = new File(directory);
+ return dir.list(new FilenameFilter() {
+ @Override
+ public boolean accept(File dir, String name) {
+ return name.startsWith("part-");
+ }
+ });
+ }
+
+ /**
+ * Create HDFS file with given content.
+ *
+ * @param directory Directory where the file should be created
+ * @param filename File name
+ * @param lines Individual lines that should be written into the file
+ * @throws IOException
+ */
+ public static void createFile(String directory, String filename, String ...lines) throws IOException {
+ File outputFile = new File(directory, filename);
+ FileUtils.writeLines(outputFile, Arrays.asList(lines));
+ }
+
+ private HdfsUtils() {
+ // Instantiation is not allowed
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java b/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
deleted file mode 100644
index fa2c2b4..0000000
--- a/test/src/test/java/org/apache/sqoop/integration/TomcatTestCase.java
+++ /dev/null
@@ -1,197 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.integration;
-
-import org.apache.commons.io.FileUtils;
-import org.apache.log4j.Logger;
-import org.apache.commons.lang.StringUtils;
-import org.apache.sqoop.client.SqoopClient;
-import org.apache.sqoop.test.minicluster.TomcatSqoopMiniCluster;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileReader;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Set;
-
-import static org.junit.Assert.fail;
-
-/**
- * Basic test case that will bootstrap Sqoop server running in external Tomcat
- * process.
- */
-abstract public class TomcatTestCase {
-
- private static final Logger LOG = Logger.getLogger(TomcatTestCase.class);
-
- @Rule public TestName name = new TestName();
-
- /**
- * Temporary base path that will be used for tests.
- *
- * By default we will take a look for sqoop.integration.tmpdir property that is
- * filled up by maven. If the test is not started from maven (IDE) we will
- * pick up configured java.io.tmpdir value. The last results is /tmp/ directory
- * in case that no property is set.
- */
- private static final String TMP_PATH_BASE =
- System.getProperty("sqoop.integration.tmpdir", System.getProperty("java.io.tmpdir", "/tmp")) + "/sqoop-cargo-tests/";
-
- /**
- * Temporary directory that will be used by the test.
- *
- * We will take TMP_PATH_BASE and append two subdirectories. First will be named
- * after fully qualified class name of current test class, second directory will
- * be named after current test method name. For example:
- *
- * TMP_PATH_BASE/org.apache.sqoop.TestClass/testMethod/
- */
- private String tmpPath;
-
- /**
- * Tomcat based Sqoop mini cluster
- */
- private TomcatSqoopMiniCluster cluster;
-
- /**
- * Sqoop client API.
- */
- private SqoopClient client;
-
- @Before
- public void startServer() throws Exception {
- // Set up the temporary path
- tmpPath = TMP_PATH_BASE + getClass().getName() + "/" + name.getMethodName() + "/";
-
- // Set up and start server
- cluster = new TomcatSqoopMiniCluster(getTemporaryPath());
- cluster.start();
-
- // Initialize Sqoop Client API
- client = new SqoopClient(getServerUrl());
- }
-
- @After
- public void stopServer() throws Exception {
- cluster.stop();
- }
-
- /**
- * Return SqoopClient configured to talk to testing server.
- *
- * @return
- */
- public SqoopClient getClient() {
- return client;
- }
-
- public String getTemporaryPath() {
- return tmpPath;
- }
-
- /**
- * Return testing server URL
- *
- * @return
- */
- public String getServerUrl() {
- return cluster.getServerUrl();
- }
-
- /**
- * Get input/output directory for mapreduce job.
- *
- * @return
- */
- public String getMapreduceDirectory() {
- return getTemporaryPath() + "/mapreduce-job-io";
- }
-
- /**
- * Return list of file names that are outputs of mapreduce job.
- *
- * @return
- */
- public String[] getOutputFilesMapreduce() {
- File dir = new File(getMapreduceDirectory());
- return dir.list(new FilenameFilter() {
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith("part-");
- }
- });
- }
-
- /**
- * Assert that mapreduce has generated following lines.
- *
- * As the lines can be spread between multiple files the ordering do not make
- * a difference.
- *
- * @param lines
- * @throws IOException
- */
- protected void assertMapreduceOutput(String... lines) throws IOException {
- Set<String> setLines = new HashSet<String>(Arrays.asList(lines));
- List<String> notFound = new LinkedList<String>();
-
- String []files = getOutputFilesMapreduce();
-
- for(String file : files) {
- String filePath = getMapreduceDirectory() + "/" + file;
- BufferedReader br = new BufferedReader(new FileReader((filePath)));
-
- String line;
- while ((line = br.readLine()) != null) {
- if (!setLines.remove(line)) {
- notFound.add(line);
- }
- }
- br.close();
- }
-
- if(!setLines.isEmpty() || !notFound.isEmpty()) {
- LOG.error("Expected lines that weren't present in the files:");
- LOG.error("\t" + StringUtils.join(setLines, "\n\t"));
- LOG.error("Extra lines in files that weren't expected:");
- LOG.error("\t" + StringUtils.join(notFound, "\n\t"));
- fail("Output do not match expectations.");
- }
- }
-
- /**
- * Create mapreduce input file with specified content.
- *
- * @param filename Input file name
- * @param lines Individual lines that should be written into the file
- * @throws IOException
- */
- protected void createInputMapreduceFile(String filename, String...lines) throws IOException {
- File outputFile = new File(getMapreduceDirectory(), filename);
- FileUtils.writeLines(outputFile, Arrays.asList(lines));
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java b/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
deleted file mode 100644
index d4e432d..0000000
--- a/test/src/test/java/org/apache/sqoop/integration/connector/ConnectorTestCase.java
+++ /dev/null
@@ -1,209 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.sqoop.integration.connector;
-
-import org.apache.log4j.Logger;
-import org.apache.sqoop.framework.configuration.OutputFormat;
-import org.apache.sqoop.framework.configuration.StorageType;
-import org.apache.sqoop.integration.TomcatTestCase;
-import org.apache.sqoop.model.MConnection;
-import org.apache.sqoop.model.MFormList;
-import org.apache.sqoop.model.MJob;
-import org.apache.sqoop.model.MPersistableEntity;
-import org.apache.sqoop.test.db.DatabaseProvider;
-import org.apache.sqoop.test.db.DatabaseProviderFactory;
-import org.apache.sqoop.validation.Status;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.fail;
-
-/**
- * Base test case for connector testing.
- *
- * It will create and initialize database provider prior every test execution.
- */
-abstract public class ConnectorTestCase extends TomcatTestCase {
-
- private static final Logger LOG = Logger.getLogger(ConnectorTestCase.class);
-
- protected static DatabaseProvider provider;
-
- @BeforeClass
- public static void startProvider() throws Exception {
- provider = DatabaseProviderFactory.getProvider(System.getProperties());
- LOG.info("Starting database provider: " + provider.getClass().getName());
- provider.start();
- }
-
- @AfterClass
- public static void stopProvider() {
- LOG.info("Stopping database provider: " + provider.getClass().getName());
- provider.stop();
- }
-
- public String getTableName() {
- return getClass().getSimpleName();
- }
-
- protected void createTable(String primaryKey, String ...columns) {
- provider.createTable(getTableName(), primaryKey, columns);
- }
-
- protected void dropTable() {
- provider.dropTable(getTableName());
- }
-
- protected void insertRow(Object ...values) {
- provider.insertRow(getTableName(), values);
- }
-
- protected long rowCount() {
- return provider.rowCount(getTableName());
- }
-
- /**
- * Fill connection form based on currently active provider.
- *
- * @param connection MConnection object to fill
- */
- protected void fillConnectionForm(MConnection connection) {
- MFormList forms = connection.getConnectorPart();
- forms.getStringInput("connection.jdbcDriver").setValue(provider.getJdbcDriver());
- forms.getStringInput("connection.connectionString").setValue(provider.getConnectionUrl());
- forms.getStringInput("connection.username").setValue(provider.getConnectionUsername());
- forms.getStringInput("connection.password").setValue(provider.getConnectionPassword());
- }
-
- /**
- * Fill output form with specific storage and output type. Mapreduce output directory
- * will be set to default test value.
- *
- * @param job MJOb object to fill
- * @param storage Storage type that should be set
- * @param output Output type that should be set
- */
- protected void fillOutputForm(MJob job, StorageType storage, OutputFormat output) {
- MFormList forms = job.getFrameworkPart();
- forms.getEnumInput("output.storageType").setValue(storage);
- forms.getEnumInput("output.outputFormat").setValue(output);
- forms.getStringInput("output.outputDirectory").setValue(getMapreduceDirectory());
- }
-
- /**
- * Fill input form. Mapreduce input directory will be set to default test value.
- *
- * @param job MJOb object to fill
- */
- protected void fillInputForm(MJob job) {
- MFormList forms = job.getFrameworkPart();
- forms.getStringInput("input.inputDirectory").setValue(getMapreduceDirectory());
- }
-
- /**
- * Create table cities.
- */
- protected void createTableCities() {
- createTable("id",
- "id", "int",
- "country", "varchar(50)",
- "city", "varchar(50)"
- );
- }
-
- /**
- * Create table cities and load few rows.
- */
- protected void createAndLoadTableCities() {
- createTableCities();
- insertRow(1, "USA", "San Francisco");
- insertRow(2, "USA", "Sunnyvale");
- insertRow(3, "Czech Republic", "Brno");
- insertRow(4, "USA", "Palo Alto");
- }
-
- /**
- * Assert row in testing table.
- *
- * @param conditions Conditions in form that are expected by the database provider
- * @param values Values that are expected in the table (with corresponding types)
- */
- protected void assertRow(Object []conditions, Object ...values) {
- ResultSet rs = provider.getRows(getTableName(), conditions);
-
- try {
- if(! rs.next()) {
- fail("No rows found.");
- }
-
- int i = 1;
- for(Object expectedValue : values) {
- Object actualValue = rs.getObject(i);
- assertEquals("Columns do not match on position: " + i, expectedValue, actualValue);
- i++;
- }
-
- if(rs.next()) {
- fail("Found more than one row.");
- }
- } catch (SQLException e) {
- LOG.error("Unexpected SQLException", e);
- fail("Unexpected SQLException: " + e);
- } finally {
- provider.closeResultSetWithStatement(rs);
- }
- }
-
- /**
- * Assert row in table "cities".
- *
- * @param values Values that are expected
- */
- protected void assertRowInCitiesTable(Object ... values) {
- assertRow(new Object[]{"id", values[0]}, values);
- }
-
- /**
- * Create connection.
- *
- * With asserts to make sure that it was created correctly.
- *
- * @param connection
- */
- protected void createConnection(MConnection connection) {
- assertEquals(Status.FINE, getClient().createConnection(connection));
- assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, connection.getPersistenceId());
- }
-
- /**
- * Create job.
- *
- * With asserts to make sure that it was created correctly.
- *
- * @param job
- */
- protected void createJob(MJob job) {
- assertEquals(Status.FINE, getClient().createJob(job));
- assertNotSame(MPersistableEntity.PERSISTANCE_ID_DEFAULT, job.getPersistenceId());
- }
-}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
index b920655..fbf46e9 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableExportTest.java
@@ -18,7 +18,7 @@
package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.log4j.Logger;
-import org.apache.sqoop.integration.connector.ConnectorTestCase;
+import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
@@ -70,10 +70,10 @@ public class TableExportTest extends ConnectorTestCase {
} while(submission.getStatus().isRunning());
assertEquals(4L, rowCount());
- assertRowInCitiesTable(1, "USA", "San Francisco");
- assertRowInCitiesTable(2, "USA", "Sunnyvale");
- assertRowInCitiesTable(3, "Czech Republic", "Brno");
- assertRowInCitiesTable(4, "USA", "Palo Alto");
+ assertRowInCities(1, "USA", "San Francisco");
+ assertRowInCities(2, "USA", "Sunnyvale");
+ assertRowInCities(3, "Czech Republic", "Brno");
+ assertRowInCities(4, "USA", "Palo Alto");
// Clean up testing table
dropTable();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
index adcfbaf..639d9ad 100644
--- a/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/connector/jdbc/generic/TableImportTest.java
@@ -20,7 +20,7 @@ package org.apache.sqoop.integration.connector.jdbc.generic;
import org.apache.log4j.Logger;
import org.apache.sqoop.framework.configuration.OutputFormat;
import org.apache.sqoop.framework.configuration.StorageType;
-import org.apache.sqoop.integration.connector.ConnectorTestCase;
+import org.apache.sqoop.test.testcases.ConnectorTestCase;
import org.apache.sqoop.model.MConnection;
import org.apache.sqoop.model.MFormList;
import org.apache.sqoop.model.MJob;
http://git-wip-us.apache.org/repos/asf/sqoop/blob/f980e90f/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
index 0f48a8b..cea24b9 100644
--- a/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/server/VersionTest.java
@@ -19,7 +19,7 @@ package org.apache.sqoop.integration.server;
import org.apache.sqoop.client.request.VersionRequest;
import org.apache.sqoop.common.VersionInfo;
-import org.apache.sqoop.integration.TomcatTestCase;
+import org.apache.sqoop.test.testcases.TomcatTestCase;
import org.apache.sqoop.json.VersionBean;
import org.junit.Test;