You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ab...@apache.org on 2014/12/22 23:57:28 UTC
sqoop git commit: SQOOP-1875: Sqoop2: Sync MSubmission with
repository and add tests for the max limit handling for submission fields
Repository: sqoop
Updated Branches:
refs/heads/sqoop2 2fdeeb06d -> 22289d236
SQOOP-1875: Sqoop2: Sync MSubmission with repository and add tests for the max limit handling for submission fields
(Veena Basavaraj via Abraham Elmahrek)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/22289d23
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/22289d23
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/22289d23
Branch: refs/heads/sqoop2
Commit: 22289d236aea48ba98d55a7478554d55056e8584
Parents: 2fdeeb0
Author: Abraham Elmahrek <ab...@apache.org>
Authored: Mon Dec 22 13:50:26 2014 -0800
Committer: Abraham Elmahrek <ab...@apache.org>
Committed: Mon Dec 22 13:50:26 2014 -0800
----------------------------------------------------------------------
...RepositoryInsertUpdateDeleteSelectQuery.java | 22 ++---
.../common/CommonRepositorySchemaConstants.java | 4 +-
.../derby/DerbyRepositoryHandler.java | 5 +-
.../repository/derby/DerbySchemaConstants.java | 7 ++
.../derby/DerbySchemaUpgradeQuery.java | 8 ++
.../sqoop/repository/derby/DerbyTestCase.java | 5 +-
.../derby/TestSubmissionHandling.java | 85 ++++++++++++++++++--
7 files changed, 113 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
index 2102db9..69c55df 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositoryInsertUpdateDeleteSelectQuery.java
@@ -390,8 +390,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ COLUMN_SQS_UPDATE_DATE + ", "
+ COLUMN_SQS_EXTERNAL_ID + ", "
+ COLUMN_SQS_EXTERNAL_LINK + ", "
- + COLUMN_SQS_EXCEPTION + ", "
- + COLUMN_SQS_EXCEPTION_TRACE + ") "
+ + COLUMN_SQS_ERROR_SUMMARY + ", "
+ + COLUMN_SQS_ERROR_DETAILS + ") "
+ " VALUES(?, ?, ?, ?, ?, ?, ?, substr(?, 1, 150) , substr(?, 1, 150), substr(?, 1, 750))";
// DML: Update existing submission
@@ -400,8 +400,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ COLUMN_SQS_STATUS + " = ?, "
+ COLUMN_SQS_UPDATE_USER + " = ?, "
+ COLUMN_SQS_UPDATE_DATE + " = ?, "
- + COLUMN_SQS_EXCEPTION + " = ?, "
- + COLUMN_SQS_EXCEPTION_TRACE + " = ?"
+ + COLUMN_SQS_ERROR_SUMMARY+ " = substr(?, 1, 150), "
+ + COLUMN_SQS_ERROR_DETAILS + " = substr(?, 1, 750)"
+ " WHERE " + COLUMN_SQS_ID + " = ?";
// DML: Check if given submission exists
@@ -428,8 +428,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ COLUMN_SQS_UPDATE_DATE + ", "
+ COLUMN_SQS_EXTERNAL_ID + ", "
+ COLUMN_SQS_EXTERNAL_LINK + ", "
- + COLUMN_SQS_EXCEPTION + ", "
- + COLUMN_SQS_EXCEPTION_TRACE
+ + COLUMN_SQS_ERROR_SUMMARY + ", "
+ + COLUMN_SQS_ERROR_DETAILS
+ " FROM " + TABLE_SQ_SUBMISSION
+ " WHERE " + COLUMN_SQS_STATUS + " = ?";
@@ -445,8 +445,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ COLUMN_SQS_UPDATE_DATE + ", "
+ COLUMN_SQS_EXTERNAL_ID + ", "
+ COLUMN_SQS_EXTERNAL_LINK + ", "
- + COLUMN_SQS_EXCEPTION + ", "
- + COLUMN_SQS_EXCEPTION_TRACE
+ + COLUMN_SQS_ERROR_SUMMARY + ", "
+ + COLUMN_SQS_ERROR_DETAILS
+ " FROM " + TABLE_SQ_SUBMISSION
+ " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC";
@@ -462,8 +462,8 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ COLUMN_SQS_UPDATE_DATE + ", "
+ COLUMN_SQS_EXTERNAL_ID + ", "
+ COLUMN_SQS_EXTERNAL_LINK + ", "
- + COLUMN_SQS_EXCEPTION + ", "
- + COLUMN_SQS_EXCEPTION_TRACE
+ + COLUMN_SQS_ERROR_SUMMARY + ", "
+ + COLUMN_SQS_ERROR_DETAILS
+ " FROM " + TABLE_SQ_SUBMISSION
+ " WHERE " + COLUMN_SQS_JOB + " = ?"
+ " ORDER BY " + COLUMN_SQS_UPDATE_DATE + " DESC";
@@ -556,4 +556,4 @@ public class CommonRepositoryInsertUpdateDeleteSelectQuery {
+ " WHERE " + COLUMN_SQ_CFG_DIR_CONFIG
+ " IN (SELECT " + COLUMN_SQ_CFG_ID + " FROM " + TABLE_SQ_CONFIG
+ " WHERE " + COLUMN_SQ_CFG_CONFIGURABLE + " = ?)";
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
index 4daf203..4ab07b2 100644
--- a/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
+++ b/repository/repository-common/src/main/java/org/apache/sqoop/repository/common/CommonRepositorySchemaConstants.java
@@ -209,9 +209,9 @@ public final class CommonRepositorySchemaConstants {
public static final String COLUMN_SQS_EXTERNAL_LINK = "SQS_EXTERNAL_LINK";
- public static final String COLUMN_SQS_EXCEPTION = "SQS_EXCEPTION";
+ public static final String COLUMN_SQS_ERROR_SUMMARY = "SQS_ERROR_SUMMARY";
- public static final String COLUMN_SQS_EXCEPTION_TRACE = "SQS_EXCEPTION_TRACE";
+ public static final String COLUMN_SQS_ERROR_DETAILS = "SQS_ERROR_DETAILS";
// SQ_COUNTER_GROUP
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
index 5bb3a43..6d35143 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbyRepositoryHandler.java
@@ -20,7 +20,6 @@ package org.apache.sqoop.repository.derby;
import static org.apache.sqoop.repository.derby.DerbySchemaCreateQuery.*;
import static org.apache.sqoop.repository.derby.DerbySchemaInsertUpdateDeleteSelectQuery.*;
import static org.apache.sqoop.repository.common.CommonRepositoryInsertUpdateDeleteSelectQuery.*;
-
import static org.apache.sqoop.repository.derby.DerbySchemaUpgradeQuery.*;
import java.net.URL;
@@ -294,6 +293,10 @@ public class DerbyRepositoryHandler extends CommonRepositoryHandler {
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIGURABLE_ID,
conn);
runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIG_ID, conn);
+ // table column rename
+ runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1, conn);
+ runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2, conn);
+
}
// last step upgrade the repository version to the latest value in the code
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
index cdb629d..a551094 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaConstants.java
@@ -327,6 +327,13 @@ public final class DerbySchemaConstants {
public static final String CONSTRAINT_SQS_SQB = SCHEMA_PREFIX + CONSTRAINT_SQS_SQB_NAME;
+ @Deprecated
+ // used only for upgrade
+ public static final String COLUMN_SQS_EXCEPTION = "SQS_EXCEPTION";
+ @Deprecated
+ // used only for upgrade
+ public static final String COLUMN_SQS_EXCEPTION_TRACE = "SQS_EXCEPTION_TRACE";
+
// SQ_COUNTER_SUBMISSION
// constraints relevant only in derby
public static final String CONSTRAINT_SQRS_SQG_NAME = CONSTRAINT_PREFIX + "SQRS_SQG";
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
index 1cdda1b..d5cabd0 100644
--- a/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
+++ b/repository/repository-derby/src/main/java/org/apache/sqoop/repository/derby/DerbySchemaUpgradeQuery.java
@@ -500,6 +500,14 @@ public final class DerbySchemaUpgradeQuery {
+ " UNIQUE ("
+ COLUMN_SQI_NAME + ", " + COLUMN_SQI_TYPE + ", " + COLUMN_SQI_CONFIG + ")";
+ // rename exception to error_summary
+ public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1 = "RENAME COLUMN "
+ + TABLE_SQ_SUBMISSION + "." + COLUMN_SQS_EXCEPTION + " TO " + COLUMN_SQS_ERROR_SUMMARY;
+
+ //rename exception_trace to error_details
+ public static final String QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2 = "RENAME COLUMN "
+ + TABLE_SQ_SUBMISSION + "." + COLUMN_SQS_EXCEPTION_TRACE + " TO " + COLUMN_SQS_ERROR_DETAILS;
+
private DerbySchemaUpgradeQuery() {
// Disable explicit object creation
}
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
index 6d4153c..6cb3eb0 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/DerbyTestCase.java
@@ -212,6 +212,9 @@ abstract public class DerbyTestCase {
runQuery(QUERY_UPGRADE_TABLE_SQ_CONFIG_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIGURABLE_ID);
// add sq_input uniqueness constraint
runQuery(QUERY_UPGRADE_TABLE_SQ_INPUT_ADD_UNIQUE_CONSTRAINT_NAME_TYPE_AND_CONFIG_ID);
+ // add submission table column name renames
+ runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_1);
+ runQuery(QUERY_UPGRADE_RENAME_TABLE_SQ_JOB_SUBMISSION_COLUMN_2);
}
// deprecated repository version
@@ -808,7 +811,7 @@ abstract public class DerbyTestCase {
runQuery("INSERT INTO SQOOP.SQ_SUBMISSION"
+ "(SQS_JOB, SQS_STATUS, SQS_CREATION_DATE, SQS_UPDATE_DATE,"
- + " SQS_EXTERNAL_ID, SQS_EXTERNAL_LINK, SQS_EXCEPTION," + " SQS_EXCEPTION_TRACE)"
+ + " SQS_EXTERNAL_ID, SQS_EXTERNAL_LINK, SQS_ERROR_SUMMARY," + " SQS_ERROR_DETAILS)"
+ "VALUES " + "(1, 'RUNNING', '2012-01-01 01:01:01', '2012-01-01 01:01:01', 'job_1',"
+ "NULL, NULL, NULL),"
+ "(2, 'SUCCEEDED', '2012-01-01 01:01:01', '2012-01-02 01:01:01', 'job_2',"
http://git-wip-us.apache.org/repos/asf/sqoop/blob/22289d23/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
----------------------------------------------------------------------
diff --git a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
index dae5e73..b479266 100644
--- a/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
+++ b/repository/repository-derby/src/test/java/org/apache/sqoop/repository/derby/TestSubmissionHandling.java
@@ -122,8 +122,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
assertEquals(1, submission.getPersistenceId());
assertCountForTable("SQOOP.SQ_SUBMISSION", 1);
- List<MSubmission> submissions =
- handler.findUnfinishedSubmissions(getDerbyDatabaseConnection());
+ List<MSubmission> submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(1, submissions.size());
@@ -166,8 +165,7 @@ public class TestSubmissionHandling extends DerbyTestCase {
assertEquals(400, counter.getValue());
// Let's create second (simpler) connection
- submission =
- new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x");
+ submission = new MSubmission(1, new Date(), SubmissionStatus.SUCCEEDED, "job-x");
handler.createSubmission(submission, getDerbyDatabaseConnection());
assertEquals(2, submission.getPersistenceId());
@@ -175,11 +173,10 @@ public class TestSubmissionHandling extends DerbyTestCase {
}
@Test
- public void testUpdateConnection() throws Exception {
+ public void testUpdateSubmission() throws Exception {
loadSubmissions();
- List<MSubmission> submissions =
- handler.findUnfinishedSubmissions(getDerbyDatabaseConnection());
+ List<MSubmission> submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection());
assertNotNull(submissions);
assertEquals(2, submissions.size());
@@ -194,6 +191,78 @@ public class TestSubmissionHandling extends DerbyTestCase {
}
@Test
+ public void testCreateSubmissionExceptionDetailsMoreThanMaxLimit() throws Exception {
+
+ String externalLink = "http://somewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresomewheresom"
+ + "ewheresomewheresomewheresomewheresomewher";
+
+ String errorSummary = "RuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptions"
+ + "RuntimeExceptionRuntimeExceptionRuntimeExceptiontests";
+ String errorDetail = "Yeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah"
+ + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it hap"
+ + "pensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYea"
+ + "h it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it ha"
+ + "ppensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah"
+ + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happe"
+ + "nsYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happens";
+ MSubmission submission = new MSubmission();
+ submission.setJobId(1);
+ submission.setStatus(SubmissionStatus.RUNNING);
+ submission.setCreationDate(new Date());
+ submission.setLastUpdateDate(new Date());
+ submission.setExternalJobId("job-x");
+ submission.setExternalLink(externalLink + "more than 150");
+ submission.getError().setErrorSummary("RuntimeException");
+ submission.getError().setErrorDetails(errorDetail + "morethan750");
+ submission.getError().setErrorSummary(errorSummary + "morethan150");
+
+ handler.createSubmission(submission, getDerbyDatabaseConnection());
+ List<MSubmission> submissions = handler.findSubmissionsForJob(1, getDerbyDatabaseConnection());
+ assertNotNull(submissions);
+
+ assertEquals(errorDetail, submissions.get(0).getError().getErrorDetails());
+ assertEquals(errorSummary, submissions.get(0).getError().getErrorSummary());
+ assertEquals(externalLink, submissions.get(0).getExternalLink());
+
+ }
+
+ @Test
+ public void testUpdateSubmissionExceptionDetailsMoreThanMaxLimit() throws Exception {
+ loadSubmissions();
+
+ List<MSubmission> submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection());
+ assertNotNull(submissions);
+ assertEquals(2, submissions.size());
+
+ String errorSummary = "RuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptionRuntimeExceptions"
+ + "RuntimeExceptionRuntimeExceptionRuntimeExceptiontests";
+
+ String errorDetail = "Yeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah"
+ + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it hap"
+ + "pensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYea"
+ + "h it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it ha"
+ + "ppensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah"
+ + " it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happe"
+ + "nsYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happensYeah it happens";
+ MSubmission submission = submissions.get(0);
+ String externalLink = submission.getExternalLink();
+ submission.getError().setErrorDetails(errorDetail + "morethan750");
+ submission.getError().setErrorSummary(errorSummary + "morethan150");
+ submission.setExternalLink("cantupdate");
+
+ handler.updateSubmission(submission, getDerbyDatabaseConnection());
+
+ submissions = handler.findUnfinishedSubmissions(getDerbyDatabaseConnection());
+
+ assertNotNull(submissions);
+ assertEquals(errorDetail, submissions.get(0).getError().getErrorDetails());
+ assertEquals(errorSummary, submissions.get(0).getError().getErrorSummary());
+ // note we dont allow external link update
+ assertEquals(externalLink, submissions.get(0).getExternalLink());
+
+ }
+
+ @Test
public void testPurgeSubmissions() throws Exception {
loadSubmissions();
List<MSubmission> submissions;
@@ -251,4 +320,4 @@ public class TestSubmissionHandling extends DerbyTestCase {
handler.deleteJob(4, getDerbyDatabaseConnection());
assertCountForTable("SQOOP.SQ_SUBMISSION", 0);
}
-}
\ No newline at end of file
+}