You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by gw...@apache.org on 2015/05/15 10:20:04 UTC
sqoop git commit: SQOOP-2356: Sqoop2: Derby upgrade should
automatically rename jobs that have conflicting names
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 d2cf56680 -> 9ee44d4f0
SQOOP-2356: Sqoop2: Derby upgrade should automatically rename jobs that have conflicting names
(Abraham Elmahrek via Gwen Shapira)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9ee44d4f
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9ee44d4f
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9ee44d4f
Branch: refs/heads/sqoop2
Commit: 9ee44d4f0a5a4601cd2fd81885b44ba08e6debfd
Parents: d2cf566
Author: Gwen Shapira <cs...@gmail.com>
Authored: Fri May 15 11:19:25 2015 +0300
Committer: Gwen Shapira <cs...@gmail.com>
Committed: Fri May 15 11:19:25 2015 +0300
----------------------------------------------------------------------
.../repository/common/CommonRepoUtils.java | 52 ++++++++
.../common/CommonRepositoryHandler.java | 13 +-
.../derby/DerbyRepositoryHandler.java | 2 +
.../derby/DerbySchemaUpgradeQuery.java | 2 +-
.../derby/upgrade/UniqueJobRename.java | 130 +++++++++++++++++++
.../derby/upgrade/Derby1_99_3UpgradeTest.java | 23 +++-
.../derby/derby-repository-1.99.3.tar.gz | Bin 119907 -> 155661 bytes
7 files changed, 207 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoUtils.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoUtils.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoUtils.java
index 96438d6..73293c0 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoUtils.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepoUtils.java
@@ -17,7 +17,14 @@
*/
package org.apache.sqoop.repository.common;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import org.apache.log4j.Logger;
+
public class CommonRepoUtils {
+ public static final Logger LOG = Logger.getLogger(CommonRepoUtils.class);
+
public static final String QUOTE_CHARACTER = "\"";
public static final String escapeTableName(String tableName) {
@@ -59,4 +66,49 @@ public class CommonRepoUtils {
return escapeConstraintName(constraintName);
}
}
+
+ /**
+ * Close all given statements.
+ *
+ * Any occurring exception is silently ignored and logged.
+ *
+ * @param stmts Statements to close
+ */
+ public static final void closeStatements(Statement ... stmts) {
+ if(stmts == null) {
+ return;
+ }
+
+ for (Statement stmt : stmts) {
+ if(stmt != null) {
+ try {
+ stmt.close();
+ } catch (SQLException ex) {
+ LOG.error("Exception during closing statement", ex);
+ }
+ }
+ }
+ }
+
+ /**
+ * Close all given Results set.
+ *
+ * Any occurring exception is silently ignored and logged.
+ *
+ * @param resultSets Result sets to close
+ */
+ public static final void closeResultSets(ResultSet... resultSets) {
+ if(resultSets == null) {
+ return;
+ }
+ for (ResultSet rs : resultSets) {
+ if(rs != null) {
+ try {
+ rs.close();
+ } catch(SQLException ex) {
+ LOG.error("Exception during closing result set", ex);
+ }
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
index 96fba2c..774c3b4 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryHandler.java
@@ -2653,18 +2653,7 @@ public abstract class CommonRepositoryHandler extends JdbcRepositoryHandler {
* @param stmts Statements to close
*/
public void closeStatements(Statement... stmts) {
- if(stmts == null) {
- return;
- }
- for (Statement stmt : stmts) {
- if(stmt != null) {
- try {
- stmt.close();
- } catch (SQLException ex) {
- LOG.error("Exception during closing statement", ex);
- }
- }
- }
+ CommonRepoUtils.closeStatements(stmts);
}
/**
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 5c247e1..2ba3384 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -54,6 +54,7 @@ import org.apache.sqoop.repository.JdbcRepositoryContext;
import org.apache.sqoop.repository.common.CommonRepoConstants;
import org.apache.sqoop.repository.common.CommonRepositoryHandler;
import org.apache.sqoop.repository.common.CommonRepositorySchemaConstants;
+import org.apache.sqoop.repository.derby.upgrade.UniqueJobRename;
/**
* JDBC based repository handler for Derby database.
@@ -276,6 +277,7 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler {
renameConnectorToConfigurable(conn);
// Add unique constraints on job and links for version 4 onwards
+ new UniqueJobRename(conn).execute();
runQuery(QUERY_UPGRADE_TABLE_SQ_JOB_ADD_UNIQUE_CONSTRAINT_NAME, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_LINK_ADD_UNIQUE_CONSTRAINT_NAME, conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIGURABLE_ADD_UNIQUE_CONSTRAINT_NAME, conn);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
index bcdb8d8..b81f81c 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
@@ -636,4 +636,4 @@ public final class DerbySchemaUpgradeQuery {
private DerbySchemaUpgradeQuery() {
// Disable explicit object creation
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/upgrade/UniqueJobRename.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/upgrade/UniqueJobRename.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/upgrade/UniqueJobRename.java
new file mode 100644
index 0000000..2322553
--- /dev/null
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/upgrade/UniqueJobRename.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sqoop.repository.derby.upgrade;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.sqoop.common.SqoopException;
+import org.apache.sqoop.error.code.DerbyRepoError;
+import org.apache.sqoop.repository.common.CommonRepoUtils;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.TreeMap;
+import java.util.UUID;
+
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_ID;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.COLUMN_SQB_NAME;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.SCHEMA_SQOOP;
+import static org.apache.sqoop.repository.common.CommonRepositorySchemaConstants.TABLE_SQ_JOB_NAME;
+
+/**
+ * Rename all jobs that have the same name to a unique name.
+ * Just provide a suffix that auto-increments.
+ */
+public class UniqueJobRename {
+ public static final String QUERY_SELECT_JOBS_WITH_NON_UNIQUE_NAMES =
+ "SELECT j1." + CommonRepoUtils.escapeColumnName(COLUMN_SQB_ID)
+ + ", j1." + CommonRepoUtils.escapeColumnName(COLUMN_SQB_NAME)
+ + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME)
+ + " AS j1 INNER JOIN ( SELECT " + CommonRepoUtils.escapeColumnName(COLUMN_SQB_NAME)
+ + " FROM " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME)
+ + " GROUP BY (" + CommonRepoUtils.escapeColumnName(COLUMN_SQB_NAME) + ")"
+ + " HAVING COUNT(*) > 1 ) AS j2 ON j1." + CommonRepoUtils.escapeColumnName(COLUMN_SQB_NAME)
+ + " = j2." + CommonRepoUtils.escapeColumnName(COLUMN_SQB_NAME);
+
+ public static final String QUERY_UPDATE_JOB_NAME_BY_ID =
+ "UPDATE " + CommonRepoUtils.getTableName(SCHEMA_SQOOP, TABLE_SQ_JOB_NAME)
+ + " SET " + CommonRepoUtils.escapeColumnName(COLUMN_SQB_NAME) + " = ?"
+ + " WHERE " + CommonRepoUtils.escapeColumnName(COLUMN_SQB_ID) + " = ?";
+
+ private Connection conn;
+
+ public UniqueJobRename(Connection conn) {
+ this.conn = conn;
+ }
+
+ public void execute() {
+ Map<Long, String> idToNewNameMap = new TreeMap<Long, String>();
+
+ Statement fetchJobStmt = null;
+ PreparedStatement updateJobStmt = null;
+ ResultSet fetchJobResultSet = null;
+
+ // Fetch all non-unique job IDs and Names.
+ // Transform names.
+ try {
+ fetchJobStmt = conn.createStatement();
+ fetchJobResultSet = fetchJobStmt.executeQuery(QUERY_SELECT_JOBS_WITH_NON_UNIQUE_NAMES);
+ while (fetchJobResultSet.next()) {
+ idToNewNameMap.put(fetchJobResultSet.getLong(1), getNewName(fetchJobResultSet.getString(2)));
+ }
+ } catch (SQLException e) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
+ } finally {
+ CommonRepoUtils.closeResultSets(fetchJobResultSet);
+ CommonRepoUtils.closeStatements(fetchJobStmt);
+ }
+
+
+ try {
+ updateJobStmt = conn.prepareStatement(QUERY_UPDATE_JOB_NAME_BY_ID);
+ for (Long jobId : idToNewNameMap.keySet()) {
+ updateJobStmt.setString(1, idToNewNameMap.get(jobId));
+ updateJobStmt.setLong(2, jobId);
+ updateJobStmt.addBatch();
+ }
+
+ int[] counts = updateJobStmt.executeBatch();
+ for (int count : counts) {
+ if (count != 1) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0000,
+ "Update count wrong when changing names for non-unique jobs. Update coutns are: "
+ + StringUtils.join(Arrays.asList(counts), ","));
+ }
+ }
+ } catch (SQLException e) {
+ throw new SqoopException(DerbyRepoError.DERBYREPO_0000, e);
+ } finally {
+ CommonRepoUtils.closeResultSets(fetchJobResultSet);
+ CommonRepoUtils.closeStatements(fetchJobStmt);
+ }
+ }
+
+ /**
+ * Make new name from old name.
+ * New name takes on form: old name + "_" + uuid.
+ * New name will substring old name if old name is longer than 47 characters.
+ * @param oldName
+ * @return newName
+ */
+ private String getNewName(String oldName) {
+ String suffix = "_" + UUID.randomUUID().toString();
+ // Make sure new name is max 64 characters.
+ int maxLength = 64 - suffix.length();
+ if (oldName.length() > maxLength) {
+ return oldName.substring(0, maxLength) + suffix;
+ } else {
+ return oldName + suffix;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
----------------------------------------------------------------------
diff --git a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
index 849ab7b..203895e 100644
--- a/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
+++ b/test/src/test/java/org/apache/sqoop/integration/repository/derby/upgrade/Derby1_99_3UpgradeTest.java
@@ -17,8 +17,16 @@
*/
package org.apache.sqoop.integration.repository.derby.upgrade;
+import org.apache.sqoop.model.MJob;
+import org.testng.annotations.Test;
+
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.AssertJUnit.assertEquals;
/**
* This version contains the following structures:
@@ -30,6 +38,8 @@ import java.util.Map;
* Job IMPORT with name "Job2" and id 2
* Job IMPORT with name "Job3" and id 3
* Job EXPORT with name "Job4" and id 4
+ * Job EXPORT with name "nonunique" and id 5
+ * Job EXPORT with name "nonunique" and id 6
* Link with id 4 has been disabled
* Job with id 3 has been disabled
* Job with id 1 has been run 5 times
@@ -48,7 +58,7 @@ public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest {
@Override
public int getNumberOfJobs() {
- return 4;
+ return 6;
}
@Override
@@ -78,6 +88,15 @@ public class Derby1_99_3UpgradeTest extends DerbyRepositoryUpgradeTest {
@Override
public Integer[] getDeleteJobIds() {
- return new Integer[] {1, 2, 3, 4};
+ return new Integer[] {1, 2, 3, 4, 5, 6};
+ }
+
+ @Test
+ public void testNonuniqueNames() throws Exception {
+ Set<String> jobNames = new TreeSet<String>();
+ for(MJob job : getClient().getJobs()) {
+ assertFalse(jobNames.contains(job.getName()));
+ jobNames.add(job.getName());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/9ee44d4f/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz
----------------------------------------------------------------------
diff --git a/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz b/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz
index bf9bf85..790d304 100644
Binary files a/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz and b/test/src/test/resources/repository/derby/derby-repository-1.99.3.tar.gz differ